Ci/docker build push (#2171)

* new minideb Dockerfile

* docker steps into CI

* project_id added to CI

* RUST_IMAGE=rust:1.80

* cargo update fix

* fix security issue with google creds in docker build step

* additional tags for docker images

* master or tag check

---------

Co-authored-by: Kirill Azovtsev <[email protected]>

.github/workflows/test.yml

@@ -38,8 +38,11 @@ env:
     ~/.hunter
     ~/.rustup
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
+  PROJECT_ID: ${{ secrets.GCP_PROJECT }}
   CACHE_PATH: ./build/cache
   DEFAULT_BUILD_TYPE: Release
+  GIT_REF_NAME: ${{ github.ref_name }}
+  IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }}
   CI: true
 
 
@@ -231,6 +234,8 @@ jobs:
             build_type: "Debug"
           - name: "Self-hosted: Minideb: Build Release"
             build_type: "Release"
+          - name: "Self-hosted: Minideb: Build RelWithDebInfo"
+            build_type: "RelWithDebInfo"
     name: "${{ matrix.options.name }}"
 
     steps:
@@ -296,6 +301,27 @@ jobs:
           make upload_apt_package \
           BUILD_TYPE=${{ matrix.options.build_type }}
 
+      - name: Set up Docker Buildx
+        if: env.package_exist == 'False'
+        uses: docker/[email protected]
+
+      - name: "Build Docker Image"
+        if: env.package_exist == 'False'
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: make kagome_image_build \
+          KAGOME_PACKAGE_VERSION=${{ env.package_version }} \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          BUILD_TYPE=${{ matrix.options.build_type }}
+
+      - name: "Push Docker Image"
+        if: env.package_exist == 'False'
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: make kagome_image_push \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          BUILD_TYPE=${{ matrix.options.build_type }} \
+          GIT_REF_NAME=${{ env.GIT_REF_NAME }} \
+          IS_MAIN_OR_TAG=${{ env.IS_MAIN_OR_TAG }}
+
       - name: "Cleaning cache"
         if: env.package_exist == 'False'
         run: |

.github/workflows/zombie-tests.yaml

@@ -29,6 +29,7 @@ on:
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
+  PROJECT_ID: ${{ secrets.GCP_PROJECT }}
   BUILD_DIR: build
   CTEST_OUTPUT_ON_FAILURE: 1
   CI: true
@@ -105,6 +106,25 @@ jobs:
           make upload_apt_package \
           BUILD_TYPE=${{ github.event.inputs.build_type }}
 
+      - name: Set up Docker Buildx
+        if: env.package_exist == 'False'
+        uses: docker/[email protected]
+
+      - name: "Build Docker Image"
+        if: env.package_exist == 'False'
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: make kagome_image_build \
+          KAGOME_PACKAGE_VERSION=${{ env.package_version }} \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          BUILD_TYPE=${{ github.event.inputs.build_type }}
+
+      - name: "Push Docker Image"
+        if: env.package_exist == 'False'
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: make kagome_image_push \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          BUILD_TYPE=${{ github.event.inputs.build_type }}
+
       - name: "Always Save Cache"
         id: cache-save
         if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')

housekeeping/docker/kagome-dev/Makefile

@@ -28,13 +28,17 @@ GITHUB_HUNTER_TOKEN ?=
 CTEST_OUTPUT_ON_FAILURE ?= 1
 WERROR ?= OFF
 
-# kagome_runtime_cache Variables
+# kagome_runtime_cache and kagome_image_build Variables
 KAGOME_PACKAGE_VERSION ?=
+#KAGOME_RUNTIME_PACKAGE_VERSION ?=
 
 # upload_apt_package Variables
 ARTIFACTS_REPO ?= kagome-apt
 REGION ?= europe-north1
 
+# CI Variables
+IS_MAIN_OR_TAG ?= false
+GIT_REF_NAME ?=
 
 export DOCKER_BUILDKIT=1
 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson
@@ -171,3 +175,40 @@ upload_apt_package_runtime:
 		gcloud artifacts versions delete $$RUNTIME_VERSION --package=kagome-dev-runtime --quiet; \
 	fi; \
 	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev-runtime_$${RUNTIME_VERSION}_$(PACKAGE_ARCHITECTURE).deb
+
+kagome_image_build:
+	$(MAKE) get_versions; \
+	SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \
+	BUILD_TYPE_LOWER=$$(echo $(BUILD_TYPE) | tr '[:upper:]' '[:lower:]'); \
+	BUILD_TARGET=""; \
+	if [ "$(BUILD_TYPE)" = "Debug" ] || [ "$(BUILD_TYPE)" = "RelWithDebInfo" ]; then \
+		BUILD_TARGET="--target debug"; \
+	fi; \
+	docker build --platform $(PLATFORM) \
+		-t $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${SHORT_COMMIT_HASH} \
+		-t $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:latest \
+		--secret id=google_creds,src=$(GOOGLE_APPLICATION_CREDENTIALS) \
+		-f minideb.Dockerfile \
+		--build-arg BASE_IMAGE=$(MINIDEB_IMAGE) \
+		--build-arg ARCHITECTURE=$(ARCHITECTURE) \
+		--build-arg KAGOME_PACKAGE_VERSION=$(KAGOME_PACKAGE_VERSION) \
+		--build-arg PROJECT_ID=$(PROJECT_ID) \
+		$${BUILD_TARGET} .
+
+kagome_image_push:
+	BUILD_TYPE_LOWER=$$(echo $(BUILD_TYPE) | tr '[:upper:]' '[:lower:]'); \
+	SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \
+	docker push $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${SHORT_COMMIT_HASH} ; \
+	docker push $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:latest ; \
+	if [ "$(IS_MAIN_OR_TAG)" = "true" ]; then \
+		if [ "$(GIT_REF_NAME)" = "master" ]; then \
+			MAIN_TAG="$${SHORT_COMMIT_HASH}-master"; \
+			docker tag $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${SHORT_COMMIT_HASH} \
+			$(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${MAIN_TAG}; \
+			docker push $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${MAIN_TAG}; \
+		elif [ -n "$(GIT_REF_NAME)" ]; then \
+			docker tag $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${SHORT_COMMIT_HASH} \
+			$(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${GIT_REF_NAME}; \
+			docker push $(DOCKER_REGISTRY_PATH)kagome_$${BUILD_TYPE_LOWER}:$${GIT_REF_NAME}; \
+		fi \
+	fi