change docker and ci build to mimic ingestion

aqden · Jan 29, 2024 · 81d704f · 81d704f
1 parent f6478c3
commit 81d704f
Show file tree

Hide file tree

Showing 2 changed files with 95 additions and 27 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -881,30 +881,83 @@ jobs:
           job-status: ${{ job.status }}
           slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel: github-activities
-  ingestapi_build:
-    name: Build and Push Ingest API image
+  ingestapi_build2:
+  # this is based on datahub_ingestion_slim_build build step. like slim, I need codegen to run successfully first. (to get the datahub.metadata classes)
+    name: Build and Push DataHub Ingestion Docker Images
     runs-on: ubuntu-latest
-    needs: setup
     outputs:
-      image_tag: ${{ steps.docker_meta.outputs.tags }}
-      image_name: ${{ env.INGEST_API_IMAGE }}
+      tag: ${{ steps.tag.outputs.tag }}
+      needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
+    needs: [setup, datahub_ingestion_base_slim_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 800
-      - name: Build and push
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            datahub-ingestion-base:
+              - 'docker/datahub-ingestion-base/**'
+            datahub-ingestion:
+              - 'docker/datahub-ingestion/**'
+      - name: Build codegen
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
+        run: ./gradlew :metadata-ingestion:codegen
+      - name: Download Base Image
+        uses: ishworkh/docker-image-artifact-download@v1
+        if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
+        with:
+          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
+      - name: Build and push IngestAPI image
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         uses: ./.github/actions/docker-custom-build-and-push
         with:
+          target: final
           images: |
-            ${{ env.INGEST_API_IMAGE }}
-          tags: ${{ needs.setup.outputs.tag }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+            ${{ env.DATAHUB_INGESTION_IMAGE }}
+          build-args: |
+            BASE_IMAGE=${{ env.INGEST_API_IMAGE }}
+            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
+            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
+            APP_ENV=slim
+          tags: ${{ needs.setup.outputs.unique_tag }}
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
           publish: ${{ needs.setup.outputs.publish }}
           context: .
           file: ./docker/ingest-api/Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64/v8
+      # - name: Compute Tag
+      #   id: tag
+      #   run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
+
+
+  # ingestapi_build:
+  #   name: Build and Push Ingest API image
+  #   runs-on: ubuntu-latest
+  #   needs: setup
+  #   outputs:
+  #     image_tag: ${{ steps.docker_meta.outputs.tags }}
+  #     image_name: ${{ env.INGEST_API_IMAGE }}
+  #   steps:
+  #     - name: Check out the repo
+  #       uses: actions/checkout@v2
+  #       with:
+  #         fetch-depth: 800
+  #     - name: Build and push
+  #       uses: ./.github/actions/docker-custom-build-and-push
+  #       with:
+  #         images: |
+  #           ${{ env.INGEST_API_IMAGE }}
+  #         tags: ${{ needs.setup.outputs.tag }}
+  #         username: ${{ secrets.DOCKER_USERNAME }}
+  #         password: ${{ secrets.DOCKER_PASSWORD }}
+  #         publish: ${{ needs.setup.outputs.publish }}
+  #         context: .
+  #         file: ./docker/ingest-api/Dockerfile
+  #         platforms: linux/amd64,linux/arm64
   ingestapi_scan:
     name: "[Monitoring] Scan ingest-api images for vulnerabilities"
     runs-on: ubuntu-latest

diff --git a/docker/ingest-api/Dockerfile b/docker/ingest-api/Dockerfile
@@ -1,23 +1,38 @@
-FROM python:3.8-slim as base
-RUN apt-get update && apt-get install -y jq
-RUN apt update && apt install -y build-essential python3-dev
+# FROM python:3.8-slim as base
+# RUN apt-get update && apt-get install -y jq
+# RUN apt update && apt install -y build-essential python3-dev
 
-# FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8-slim as base
-FROM openjdk:8 as prod-build
-COPY . /datahub-src
-RUN cd /datahub-src && ./gradlew :metadata-ingestion:codegen -d
+# # FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8-slim as base
+# FROM openjdk:8 as prod-build
+# COPY . /datahub-src
+# RUN cd /datahub-src && ./gradlew :metadata-ingestion:codegen -d
 
-FROM base as prod-codegen
-COPY --from=prod-build /datahub-src /datahub-src
-RUN cd /datahub-src/metadata-ingestion && \
-    pip install -e ".[base]" 
+# FROM base as prod-codegen
+# COPY --from=prod-build /datahub-src /datahub-src
+# RUN cd /datahub-src/metadata-ingestion && \
+#     pip install -e ".[base]" 
 
-FROM base as final
+# FROM base as final
+# ENV PROMETHEUS_MULTIPROC_DIR=/gunicorn/
+# COPY --from=prod-codegen /datahub-src/metadata-ingestion /datahub-ingestion
+# RUN pip install /datahub-ingestion/.
+# COPY ./ingest-api/ /ingest-api/
+# WORKDIR /ingest-api/ingest_api
+# RUN pip install -r ../requirements.txt && pip install ../ && mkdir /gunicorn/
+# CMD ["gunicorn", "-c", "config.py", "main:app", "--workers", "4", "--worker-class", "uvicorn.workers.UvicornWorker", "--bind", "0.0.0.0:8001"]
+
+ARG APP_ENV=full
+ARG BASE_IMAGE=acryldata/datahub-ingestion-base
+ARG DOCKER_VERSION=head
+
+FROM $BASE_IMAGE:$DOCKER_VERSION as base
+USER 0
 ENV PROMETHEUS_MULTIPROC_DIR=/gunicorn/
-COPY --from=prod-codegen /datahub-src/metadata-ingestion /datahub-ingestion
-RUN pip install /datahub-ingestion/.
+COPY ./metadata-ingestion /datahub-ingestion
+WORKDIR /datahub-ingestion
+RUN pip install --no-cache --user ".[base]"
+
 COPY ./ingest-api/ /ingest-api/
 WORKDIR /ingest-api/ingest_api
 RUN pip install -r ../requirements.txt && pip install ../ && mkdir /gunicorn/
 CMD ["gunicorn", "-c", "config.py", "main:app", "--workers", "4", "--worker-class", "uvicorn.workers.UvicornWorker", "--bind", "0.0.0.0:8001"]
-