publish builder gpu image (milvus-io#27861)

Signed-off-by: PowderLi <[email protected]>
congqixia · Oct 25, 2023 · a951c75 · a951c75
1 parent 53246b1
commit a951c75
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 3 deletions.
diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml
@@ -7,12 +7,14 @@ on:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
+      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
   pull_request:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
+      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
 
@@ -93,3 +95,81 @@ jobs:
         run: |
           echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
           echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
+
+
+  publish-gpu-builder:
+    name: ${{ matrix.arch }} ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu20.04 ]
+        arch: [ amd64&arm64 ]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=version::$(date +%Y%m%d)"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - name: Docker Pull
+        shell: bash
+        run: |
+          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
+          docker buildx ls
+          docker buildx create --use --name=milvus --driver docker-container
+      - name: Docker Build
+        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Docker Build&Push
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        continue-on-error: true
+        shell: bash
+        run: |
+          docker buildx ls
+          docker login -u ${{ secrets.DOCKERHUB_USER }} \
+                       -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Update Builder Image Changes
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        continue-on-error: true
+        shell: bash
+        run: |
+          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add .env
+          git commit -m "Update Builder gpu image changes"
+      - name: Create Pull Request
+        id: cpr
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: update_builder_${{ github.sha }}
+          delete-branch: true
+          title: '[automated] Update Builder gpu image changes'
+          body: |
+            Update Builder gpu image changes
+            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
+            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
+      - name: Check outputs
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/build/docker/builder/gpu/ubuntu20.04/Dockerfile b/build/docker/builder/gpu/ubuntu20.04/Dockerfile
@@ -11,6 +11,8 @@
 
 FROM nvidia/cuda:11.8.0-devel-ubuntu20.04
 
+ARG TARGETARCH
+
 RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 ninja-build && \
     wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
     apt-get update && apt-get install -y --no-install-recommends \
@@ -39,9 +41,9 @@ ENV GOPATH /go
 ENV GOROOT /usr/local/go
 ENV GO111MODULE on
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.20.4.linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
+RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.20.7.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
     mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
-    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.46.2 && \
+    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOROOT}/bin v1.46.2 && \
     # export GO111MODULE=on && go get github.com/quasilyte/go-ruleguard/cmd/[email protected] && \
     go install github.com/ramya-rao-a/go-outline@latest && \
     go install golang.org/x/tools/gopls@latest && \

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -53,7 +53,7 @@ services:
       context: .
       dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile
       cache_from:
-        - ${IMAGE_REPO}/milvus-env:$${OS_NAME}-${LATEST_DATE_VERSION}
+        - ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${LATEST_GPU_DATE_VERSION}
     # user: {{ CURRENT_ID }}
     shm_size: 2G
     # expose 19530 port so we can directly access milvus inside build container