add conditional build datahub-ingestion unified

.github/workflows/docker-unified.yml

@@ -420,13 +420,26 @@ jobs:
   datahub_ingestion_build:
     name: Build and Push DataHub Ingestion Docker Images
     runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
     needs: setup
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
           fetch-depth: 800
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            datahub-ingestion-base:
+              - 'docker/datahub-ingestion-base/**'
+            datahub-ingestion:
+              - 'docker/datahub-ingestion/**'
+            datahub-ingestion-slim:
+              - 'docker/datahub-ingestion-slim/**'
       - name: Build and push Base Image
+        if: steps.filter.outputs.datahub-ingestion-base == 'true'
         uses: ./.github/actions/docker-custom-build-and-push
         with:
           images: |
@@ -439,14 +452,16 @@ jobs:
           file: ./docker/datahub-ingestion-base/Dockerfile
           platforms: linux/amd64,linux/arm64/v8
       - name: Docker Prune
+        if: steps.filter.outputs.datahub-ingestion-base == 'true'
         run: docker image prune -a -f
       - name: Build and push Full Image
+        if: steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true'
         uses: ./.github/actions/docker-custom-build-and-push
         with:
           images: |
             ${{ env.DATAHUB_INGESTION_IMAGE }}
           build-args: |
-            DOCKER_VERSION=${{ needs.setup.outputs.tag }}
+            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.tag || 'head' }}
           tags: ${{ needs.setup.outputs.tag }}
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
@@ -455,21 +470,27 @@ jobs:
           file: ./docker/datahub-ingestion/Dockerfile
           platforms: linux/amd64,linux/arm64/v8
       - name: Docker Prune
+        if: steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true'
         run: docker image prune -a -f
       - name: Build and push Slim Image
+        if: steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true'
         uses: ./.github/actions/docker-custom-build-and-push
         with:
           images: |
             ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}
           build-args: |
-            DOCKER_VERSION=${{ needs.setup.outputs.tag }}
+            DOCKER_VERSION=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.tag || 'head' }}
           tags: ${{ needs.setup.outputs.tag }}
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
           publish: ${{ needs.setup.outputs.publish }}
           context: .
           file: ./docker/datahub-ingestion-slim/Dockerfile
           platforms: linux/amd64,linux/arm64/v8
+      - name: Compute Tag
+        id: tag
+        run: |
+          echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_scan:
     permissions:
       contents: read # for actions/checkout to fetch code
@@ -481,51 +502,36 @@ jobs:
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: actions/checkout@v3
-      - name: Download image Base Image
-        uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
-        with:
-          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download image Full Image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
-          image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-      - name: Download image Slim Image
-        uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
-        with:
-          image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-      - name: Run Trivy vulnerability scanner Base Image
-        uses: aquasecurity/[email protected]
-        env:
-          TRIVY_OFFLINE_SCAN: true
-        with:
-          image-ref: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-          format: "template"
-          template: "@/contrib/sarif.tpl"
-          output: "trivy-results.sarif"
-          severity: "CRITICAL,HIGH"
-          ignore-unfixed: true
-          vuln-type: "os,library"
+          image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Full Image
         uses: aquasecurity/[email protected]
         env:
           TRIVY_OFFLINE_SCAN: true
         with:
-          image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+          image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
           format: "template"
           template: "@/contrib/sarif.tpl"
           output: "trivy-results.sarif"
           severity: "CRITICAL,HIGH"
           ignore-unfixed: true
           vuln-type: "os,library"
+      - name: Docker Prune
+        run: docker image prune -a -f
+      - name: Download image Slim Image
+        uses: ishworkh/docker-image-artifact-download@v1
+        if: ${{ needs.setup.outputs.publish != 'true' }}
+        with:
+          image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Slim Image
         uses: aquasecurity/[email protected]
         env:
           TRIVY_OFFLINE_SCAN: true
         with:
-          image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+          image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
           format: "template"
           template: "@/contrib/sarif.tpl"
           output: "trivy-results.sarif"
@@ -638,7 +644,7 @@ jobs:
           DATAHUB_TELEMETRY_ENABLED: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}
-          ACTIONS_VERSION: ${{ needs.setup.outputs.unique_tag }}
+          ACTIONS_VERSION: ${{ needs.datahub_ingestion_build.outputs.tag }}
           ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions'
           ACTIONS_CONFIG: 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
         run: |
@@ -648,6 +654,18 @@ jobs:
           # we are doing this because gms takes time to get ready
           # and we don't have a better readiness check when bootstrap is done
           sleep 60s
+      - name: Disable ES Disk Threshold
+        run: |
+          curl -XPUT "http://localhost:9200/_cluster/settings" \
+          -H 'Content-Type: application/json' -d'{
+            "persistent": {
+              "cluster": {
+                "routing": {
+                  "allocation.disk.threshold_enabled": false
+                }
+              }
+            }
+          }'
       - name: Disk Check
         run: df -h .
       - name: Docker Image Check