diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 1570dcc6daac9..2aafccde845f2 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -881,30 +881,83 @@ jobs: job-status: ${{ job.status }} slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} channel: github-activities - ingestapi_build: - name: Build and Push Ingest API image + ingestapi_build2: + # this is based on datahub_ingestion_slim_build build step. like slim, I need codegen to run successfully first. (to get the datahub.metadata classes) + name: Build and Push DataHub Ingestion Docker Images runs-on: ubuntu-latest - needs: setup outputs: - image_tag: ${{ steps.docker_meta.outputs.tags }} - image_name: ${{ env.INGEST_API_IMAGE }} + tag: ${{ steps.tag.outputs.tag }} + needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }} + needs: [setup, datahub_ingestion_base_slim_build] steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 800 - - name: Build and push + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + datahub-ingestion-base: + - 'docker/datahub-ingestion-base/**' + datahub-ingestion: + - 'docker/datahub-ingestion/**' + - name: Build codegen + if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} + run: ./gradlew :metadata-ingestion:codegen + - name: Download Base Image + uses: ishworkh/docker-image-artifact-download@v1 + if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} + with: + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} + - name: Build and push IngestAPI image + if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} uses: ./.github/actions/docker-custom-build-and-push with: + target: final images: | - ${{ env.INGEST_API_IMAGE }} - tags: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + ${{ env.DATAHUB_INGESTION_IMAGE }} + build-args: | + BASE_IMAGE=${{ env.INGEST_API_IMAGE }} + DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} + RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} + APP_ENV=slim + tags: ${{ needs.setup.outputs.unique_tag }} + username: ${{ secrets.ACRYL_DOCKER_USERNAME }} + password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/ingest-api/Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64/v8 + # - name: Compute Tag + # id: tag + # run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT + + + # ingestapi_build: + # name: Build and Push Ingest API image + # runs-on: ubuntu-latest + # needs: setup + # outputs: + # image_tag: ${{ steps.docker_meta.outputs.tags }} + # image_name: ${{ env.INGEST_API_IMAGE }} + # steps: + # - name: Check out the repo + # uses: actions/checkout@v2 + # with: + # fetch-depth: 800 + # - name: Build and push + # uses: ./.github/actions/docker-custom-build-and-push + # with: + # images: | + # ${{ env.INGEST_API_IMAGE }} + # tags: ${{ needs.setup.outputs.tag }} + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # publish: ${{ needs.setup.outputs.publish }} + # context: . + # file: ./docker/ingest-api/Dockerfile + # platforms: linux/amd64,linux/arm64 ingestapi_scan: name: "[Monitoring] Scan ingest-api images for vulnerabilities" runs-on: ubuntu-latest diff --git a/docker/ingest-api/Dockerfile b/docker/ingest-api/Dockerfile index 3656e38fea8f0..0d23852e94734 100644 --- a/docker/ingest-api/Dockerfile +++ b/docker/ingest-api/Dockerfile @@ -1,23 +1,38 @@ -FROM python:3.8-slim as base -RUN apt-get update && apt-get install -y jq -RUN apt update && apt install -y build-essential python3-dev +# FROM python:3.8-slim as base +# RUN apt-get update && apt-get install -y jq +# RUN apt update && apt install -y build-essential python3-dev -# FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8-slim as base -FROM openjdk:8 as prod-build -COPY . /datahub-src -RUN cd /datahub-src && ./gradlew :metadata-ingestion:codegen -d +# # FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8-slim as base +# FROM openjdk:8 as prod-build +# COPY . /datahub-src +# RUN cd /datahub-src && ./gradlew :metadata-ingestion:codegen -d -FROM base as prod-codegen -COPY --from=prod-build /datahub-src /datahub-src -RUN cd /datahub-src/metadata-ingestion && \ - pip install -e ".[base]" +# FROM base as prod-codegen +# COPY --from=prod-build /datahub-src /datahub-src +# RUN cd /datahub-src/metadata-ingestion && \ +# pip install -e ".[base]" -FROM base as final +# FROM base as final +# ENV PROMETHEUS_MULTIPROC_DIR=/gunicorn/ +# COPY --from=prod-codegen /datahub-src/metadata-ingestion /datahub-ingestion +# RUN pip install /datahub-ingestion/. +# COPY ./ingest-api/ /ingest-api/ +# WORKDIR /ingest-api/ingest_api +# RUN pip install -r ../requirements.txt && pip install ../ && mkdir /gunicorn/ +# CMD ["gunicorn", "-c", "config.py", "main:app", "--workers", "4", "--worker-class", "uvicorn.workers.UvicornWorker", "--bind", "0.0.0.0:8001"] + +ARG APP_ENV=full +ARG BASE_IMAGE=acryldata/datahub-ingestion-base +ARG DOCKER_VERSION=head + +FROM $BASE_IMAGE:$DOCKER_VERSION as base +USER 0 ENV PROMETHEUS_MULTIPROC_DIR=/gunicorn/ -COPY --from=prod-codegen /datahub-src/metadata-ingestion /datahub-ingestion -RUN pip install /datahub-ingestion/. +COPY ./metadata-ingestion /datahub-ingestion +WORKDIR /datahub-ingestion +RUN pip install --no-cache --user ".[base]" + COPY ./ingest-api/ /ingest-api/ WORKDIR /ingest-api/ingest_api RUN pip install -r ../requirements.txt && pip install ../ && mkdir /gunicorn/ CMD ["gunicorn", "-c", "config.py", "main:app", "--workers", "4", "--worker-class", "uvicorn.workers.UvicornWorker", "--bind", "0.0.0.0:8001"] -