Merge tag 'v0.10.5' into filesink_json

* tag 'v0.10.5': (222 commits)
  fix(test): increase siblings.js test stability (datahub-project#8542)
  feat(search): Allow aggregating on facets that are not explicitly part of default filter set (datahub-project#8540)
  fix(ui) Make multiple small updates to new search and browse (datahub-project#8524)
  feat(presto-on-hive): allow v1 fieldpaths in the presto-on-hive source (datahub-project#8474)
  feat(cli): Adds ability to upload recipes to DataHub's UI (datahub-project#8317)
  feat(browseV2): add browseV2 logic to system update (datahub-project#8506)
  fix(ingest/json-schema): convert non-string enums to strings (datahub-project#8479)
  feat(ingestion/tableau): support column level lineage for custom sql (datahub-project#8466)
  test(ingest): test case statements with sql parser (datahub-project#8437)
  feat(ingest/vertica): performance improvement and bug fixes (datahub-project#8328)
  ci: reduce git fetch depth (datahub-project#8473)
  fix(ingest): remove duplication of tags (datahub-project#8532)
  docs: small update to homepage (datahub-project#8483)
  fix(ingest): pin boto3-stubs in CI (datahub-project#8527)
  feat(siblings): hiding non-existant siblings in FE (datahub-project#8528)
  fix(ingest/build): Fix sagemaker mypy and flake8 issues (datahub-project#8530)
  feat(metrics): add metrics for aspect write and bytes (datahub-project#8526)
  feat(elasticsearch): allow bulk delete (datahub-project#8424)
  fix(ui): use locale lowercase when filtering columns of an entity in the lineage (datahub-project#8213)
  fix(auth): ignore case when comparing http headers (datahub-project#8356)
  ...

.github/scripts/check_event_type.py

@@ -1,7 +1,7 @@
 import sys
 
 java_events = set()
-with open("./metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
+with open("./metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
     for line in java_file:
         if '''Event"''' not in line:
             continue

.github/workflows/build-and-test.yml

@@ -43,7 +43,7 @@ jobs:
           timezoneLinux: ${{ matrix.timezone }}
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

.github/workflows/check-datahub-jars.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

.github/workflows/code-checks.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/docker-ingestion-base.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and Push image
         uses: ./.github/actions/docker-custom-build-and-push
         with:

.github/workflows/docker-ingestion-smoke.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

.github/workflows/docker-ingestion.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

.github/workflows/docker-postgres-setup.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

.github/workflows/docker-unified.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-service:war:build -x test --parallel
@@ -122,7 +122,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
@@ -180,7 +180,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
@@ -238,7 +238,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-upgrade:build -x test --parallel
@@ -296,7 +296,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
@@ -356,7 +356,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -378,7 +378,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -400,7 +400,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -490,6 +490,9 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: Disable datahub-actions
+        run: |
+          yq -i 'del(.services.datahub-actions)' docker/quickstart/docker-compose-without-neo4j.quickstart.yml
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false
@@ -501,6 +504,20 @@ jobs:
           # we are doing this because gms takes time to get ready
           # and we don't have a better readiness check when bootstrap is done
           sleep 60s
+      - name: Disable ES Disk Threshold
+        run: |
+          curl -XPUT "http://localhost:9200/_cluster/settings" \
+          -H 'Content-Type: application/json' -d'{
+            "persistent": {
+              "cluster": {
+                "routing": {
+                  "allocation.disk.threshold_enabled": false
+                }
+              }
+            }
+          }'
+      - name: Remove Source Code
+        run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete
       - name: Smoke test
         env:
           RUN_QUICKSTART: false

.github/workflows/publish-datahub-jars.yml

@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

build.gradle

@@ -129,6 +129,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20230227',
+    'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
     'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",

datahub-frontend/app/controllers/Application.java

@@ -136,11 +136,11 @@ public CompletableFuture<Result> proxy(String path, Http.Request request) throws
             .stream()
             // Remove X-DataHub-Actor to prevent malicious delegation.
             .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equalsIgnoreCase(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equals(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equalsIgnoreCase(entry.getKey()))
             // Remove Host s.th. service meshes do not route to wrong host
-            .filter(entry -> !Http.HeaderNames.HOST.equals(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.HOST.equalsIgnoreCase(entry.getKey()))
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
         )
         .addHeader(Http.HeaderNames.AUTHORIZATION, authorizationHeaderValue)
@@ -152,8 +152,8 @@ public CompletableFuture<Result> proxy(String path, Http.Request request) throws
           final ResponseHeader header = new ResponseHeader(apiResponse.getStatus(), apiResponse.getHeaders()
               .entrySet()
               .stream()
-              .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey()))
-              .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey()))
+              .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey()))
+              .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey()))
               .map(entry -> Pair.of(entry.getKey(), String.join(";", entry.getValue())))
               .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)));
           final HttpEntity body = new HttpEntity.Strict(apiResponse.getBodyAsBytes(), Optional.ofNullable(apiResponse.getContentType()));

datahub-graphql-core/build.gradle

@@ -7,6 +7,8 @@ dependencies {
     compile project(':metadata-service:restli-client')
     compile project(':metadata-service:auth-impl')
     compile project(':metadata-service:auth-config')
+    compile project(':metadata-service:configuration')
+    compile project(':metadata-service:services')
     compile project(':metadata-io')
     compile project(':metadata-utils')
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -21,6 +21,7 @@ public class Constants {
     public static final String STEPS_SCHEMA_FILE = "step.graphql";
     public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
     public static final String BROWSE_PATH_DELIMITER = "/";
+    public static final String BROWSE_PATH_V2_DELIMITER = "␟";
     public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
 
     public static final String ENTITY_FILTER_NAME = "_entityType";