Build images in GitHub Actions

.github/workflows/images.yaml

@@ -0,0 +1,61 @@
+name: Images
+on:
+  push:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    name: Build and push all images
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Register binfmt from multi-platform builds
+        with:
+          entrypoint: docker
+          args: run --privileged linuxkit/binfmt:5d33e7346e79f9c13a73c6952669e47a53b063d4 
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make lint
+        with:
+          entrypoint: make
+          args: lint
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make maker-image
+        env:
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        with:
+          entrypoint: make
+          args: maker-image PUSH=true
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make compilers-image
+        env:
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        with:
+          entrypoint: make
+          args: compilers-image PUSH=true
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make bpftool-image
+        env:
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        with:
+          entrypoint: make
+          args: bpftool-image PUSH=true
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make iproute2-image
+        env:
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        with:
+          entrypoint: make
+          args: iproute2-image PUSH=true
+      - uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make llvm-image
+        env:
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        with:
+          entrypoint: make
+          args: llvm-image PUSH=true

.github/workflows/pr-checks.yaml

@@ -0,0 +1,14 @@
+name: PR Check
+on: [pull_request,push]
+
+jobs:
+  lint:
+    name: Run static checks
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker://docker.io/cilium/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef
+        name: Run make lint
+        with:
+          entrypoint: make
+          args: lint

.gitignore

@@ -0,0 +1,2 @@
+.buildx_builder
+.buildx

.hadolint.yaml

@@ -0,0 +1,8 @@
+ignored:
+# it cannot parse `FROM ${BASE_IMAGE}`
+- DL3006
+# alpine doesn't keep old versions around, so we cannot rely on package pinning
+- DL3018
+# this one just needs to be set, but since `FROM ${BASE_IMAGE}` cannot be parse,
+# there isn't a way to leverage this feature anyway
+allowedRegistries: []

LICENSE

@@ -0,0 +1,13 @@
+Copyright 2020 Authors of Cilium. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

Makefile

@@ -0,0 +1,40 @@
+# Copyright 2020 Authors of Cilium
+# SPDX-License-Identifier: Apache-2.0
+
+REGISTRY_PREFIX ?= docker.io/cilium
+PUSH ?= false
+
+OUTPUT := "type=docker"
+ifeq ($(PUSH),true)
+OUTPUT := "type=registry,push=true"
+endif
+
+all-images: lint maker-image update-maker-image compilers-image update-compilers-image bpftool-image iproute2-image llvm-image
+
+lint:
+	scripts/lint.sh
+
+.buildx_builder:
+	mkdir -p .buildx
+	docker buildx create --platform linux/amd64,linux/arm64 > $@
+
+maker-image: .buildx_builder
+	scripts/build-image.sh $(REGISTRY_PREFIX)/image-maker images/maker linux/amd64 $(OUTPUT) "$$(cat .buildx_builder)"
+
+update-maker-image:
+	scripts/update-maker-image.sh $(REGISTRY_PREFIX)
+
+compilers-image: .buildx_builder
+	scripts/build-image.sh $(REGISTRY_PREFIX)/image-compilers images/compilers linux/amd64 $(OUTPUT) "$$(cat .buildx_builder)"
+
+update-compilers-image:
+	scripts/update-compilers-image.sh $(REGISTRY_PREFIX)
+
+bpftool-image: .buildx_builder
+	scripts/build-image.sh $(REGISTRY_PREFIX)/bpftool images/bpftool linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)"
+
+iproute2-image: .buildx_builder
+	scripts/build-image.sh $(REGISTRY_PREFIX)/iproute2 images/iproute2 linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)"
+
+llvm-image: .buildx_builder
+	scripts/build-image.sh $(REGISTRY_PREFIX)/llvm images/llvm linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)"

README.md

@@ -0,0 +1,100 @@
+# Cilium Dependency Packaging
+
+This repository contains build definitions for a number of images that are components of the official and development images of Cilium.
+
+The builds are currently hosted in GitHub Actions, but can be ported to any other container-based CI system.
+
+## Images
+
+### [`images/maker`](images/maker/Dockerfile)
+
+This image consists of core tools used for building all other images, which include `bash`, `make` and `docker` (with [`buildx`](https://github.com/docker/buildx))
+and [`crane`](https://github.com/google/go-containerregistry/blob/master/cmd/crane).
+This image enables using latest BuildKit features without depending on whatever Docker daemon/client CI host provides.
+Since `buildx` runs a BuildKit daemon inside a container, it's largely independent of what version of Docker daemon it runs on.
+
+This image also includes a secure credentials helper - [`docker-credential-env`](http://github.com/errordeveloper/docker-credential-env),
+which prevents having to use `docker login` which stores a plain text token in `${DOCKER_CONFIG}/config.json`.
+
+### [`images/compiler`](images/compilers/Dockerfile)
+
+This image consists of compilers and libraries needed to build other images for `amd64` and `arm64`.
+
+### [`images/bpftool`](images/bpftool/Dockerfile)
+
+This image builds `bpftool` binary for `amd64` and `arm64` using a cross-compiler. The resulting image has only one file -
+`/bin/bpftool`, it is a proper multi-platform image. The binary is dynamically linked to Ubuntu 20.04 glibc and other dependencies.
+
+This image is uses a recent version of `bpftool` from `bpf-next` Linux kernel tree.
+
+### [`images/iproute2`](images/iproute2/Dockerfile)
+
+This image builds `ip` and `tc` binaries for `amd64` and `arm64` using a cross-compiler. The resulting image has only two files -
+`/bin/ip` and `/bin/tc`, it is a proper multi-platform image. The binaries are dynamically linked to Ubuntu 20.04 glibc and other
+dependencies.
+
+This image is uses [a fork of `iproute2`](https://github.com/cilium/iproute2), it has features that Cilium relies on.
+
+### [`images/llvm`](images/llvm/Dockerfile)
+
+This image builds `llc` and `clang` binaries for `amd64` and `arm64` using a cross-compiler. The resulting image has only two
+files - `/bin/llc` and `/bin/clang`, it is a proper multi-platform image. The binaries are dynamically linked to Ubuntu 20.04 glibc
+and other dependencies.
+
+This image is a custom BPF-only distribution of LLVM.
+
+## Usage
+
+### Making changes
+
+All images get automatic tags based on checked-in contents of image subdirectory. At any point in git history of a subdirectory
+there exists a unique [git tree object hash](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), that is what's used for
+image tags.
+
+As the result of this, following stands:
+
+- image build definitions can be obtained with `git show <tag>`
+- image build is defined by contents of a directory
+- when changes are committed to image directory, new tag is generated
+    - if there is a new tag, image is rebuilt and pushed with that new tag
+
+This does not cater for reproducible builds, however it serves as basis for reliable builds, especially when following rules
+are also applied to any build definitions:
+
+- all `FROM` statements use digests (use `scripts/get-image-digest.sh`)
+- any system packages are installed in a separate image that is references by a digests (that's how `images/compilers` is designed)
+    - pining system packages can be quite laborious, especially because most of the time what you want is latest that the distribution offers,
+      so what's much easier to let the package manager get the latest and then pin down the result by digest, so every time there is a change
+      in underlying system packages, that is explicitly recorded by change of digest in each image that uses the base image
+
+Be sure to use `make lint`, which will run [`shellcheck`](https://github.com/koalaman/shellcheck) and [`hadolint`](https://github.com/hadolint/hadolint).
+
+For details of how this works, see the following:
+
+- [`Makefile`](Makefile)
+- [`scripts/build-image.sh`](`scripts/build-image.sh`)
+- [`scripts/make-image-tag.sh`](scripts/make-image-tag.sh).
+- [`images/maker`](images/maker/Dockerfile)
+- [`images/compilers`](images/compilers/Dockerfile)
+
+### Building Locally
+
+One should be able to build images locally as long as they have Docker installed with [`buildx` plug-in](https://docs.docker.com/buildx/working-with-buildx/).
+
+#### `images/{maker,compilers}`
+
+When you have dependencies that need to be added to these image before using them in one of the other images, e.g. if you need to add a system
+library in `compilers` image that will be used for compiling something else, you should make a PR to update `compilers` first.
+However, that's only required for full integration, and you can build images locally if you prefer, you can also push them to your own Docker Hub
+account or whatever is your preferred registry.
+
+When changes to these images are merged into master, builds should run and push new images to each of the registries.
+Once new images are out, a PR will be required to update all dependent images, please use the following commands to
+make updates and commit the resulting changes:
+
+> NOTE: You can only use the `update-*-image` make targets when you are _not building localy_. For example, if you have built a new `compilers`
+> locally, or perahps even pushed it your Docker Hyb account, and you want to consume this new version to build new `llvm` image - you need to uptade
+> `images/llvm/Dockerfile` manually, as the `scripts/update-*-images.sh` is not capable of handling this.
+
+- `make update-maker-image`
+- `make update-compilers-image`

images/bpftool/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

images/bpftool/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2020 Authors of Cilium
+# SPDX-License-Identifier: Apache-2.0
+
+ARG COMPILERS_IMAGE=docker.io/cilium/image-compilers:818f48aa3eada1d23018f02f3c43a169c51eea1a@sha256:54e7c0d8713f479e578dbcfd915795db9b12761d6397bf7ef61ae4b1d2031f03
+
+FROM --platform=linux/amd64 ${COMPILERS_IMAGE} as builder
+
+COPY checkout-linux.sh /tmp/checkout-linux.sh
+RUN /tmp/checkout-linux.sh
+
+COPY build-bpftool-native.sh /tmp/build-bpftool-native.sh
+RUN /tmp/build-bpftool-native.sh
+
+COPY build-bpftool-cross-aarch64.sh /tmp/build-bpftool-cross-aarch64.sh
+RUN /tmp/build-bpftool-cross-aarch64.sh
+
+FROM scratch
+LABEL maintainer="[email protected]"
+ARG TARGETPLATFORM
+COPY --from=builder /out/${TARGETPLATFORM}/bin /bin

images/bpftool/build-bpftool-cross-aarch64.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2017-2020 Authors of Cilium
+# SPDX-License-Identifier: Apache-2.0
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+set -o nounset
+
+triplet="aarch64-linux-gnu"
+
+cd /src/linux/tools/bpf/bpftool
+
+make clean
+
+make -j "$(getconf _NPROCESSORS_ONLN)" ARCH=arm64 CROSS_COMPILE=${triplet}-
+
+${triplet}-strip bpftool
+
+mkdir -p /out/linux/arm64/bin
+cp bpftool /out/linux/arm64/bin

images/bpftool/build-bpftool-native.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2017-2020 Authors of Cilium
+# SPDX-License-Identifier: Apache-2.0
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+set -o nounset
+
+cd /src/linux/tools/bpf/bpftool
+
+make -j "$(getconf _NPROCESSORS_ONLN)"
+
+strip bpftool
+
+mkdir -p /out/linux/amd64/bin
+cp bpftool /out/linux/amd64/bin