-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
7877cb0
commit 61d38f6
Showing
37 changed files
with
902 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
name: Images | ||
on: | ||
push: | ||
branches: [master] | ||
|
||
jobs: | ||
build-and-push: | ||
name: Build and push all images | ||
runs-on: ubuntu-18.04 | ||
steps: | ||
- uses: actions/checkout@v1 | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Register binfmt from multi-platform builds | ||
with: | ||
entrypoint: docker | ||
args: run --privileged linuxkit/binfmt:5d33e7346e79f9c13a73c6952669e47a53b063d4 | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make lint | ||
with: | ||
entrypoint: make | ||
args: lint | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make maker-image | ||
env: | ||
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
with: | ||
entrypoint: make | ||
args: maker-image PUSH=true | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make compilers-image | ||
env: | ||
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
with: | ||
entrypoint: make | ||
args: compilers-image PUSH=true | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make bpftool-image | ||
env: | ||
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
with: | ||
entrypoint: make | ||
args: bpftool-image PUSH=true | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make iproute2-image | ||
env: | ||
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
with: | ||
entrypoint: make | ||
args: iproute2-image PUSH=true | ||
- uses: docker://docker.io/errordeveloper/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make llvm-image | ||
env: | ||
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
with: | ||
entrypoint: make | ||
args: llvm-image PUSH=true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
name: PR Check | ||
on: [pull_request,push] | ||
|
||
jobs: | ||
lint: | ||
name: Run static checks | ||
runs-on: ubuntu-18.04 | ||
steps: | ||
- uses: actions/checkout@v1 | ||
- uses: docker://docker.io/cilium/image-maker:2831b3fa8bc8a1412ed8eb59b158a123fe0459ef | ||
name: Run make lint | ||
with: | ||
entrypoint: make | ||
args: lint |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
.buildx_builder | ||
.buildx |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
ignored: | ||
# it cannot parse `FROM ${BASE_IMAGE}` | ||
- DL3006 | ||
# alpine doesn't keep old versions around, so we cannot rely on package pinning | ||
- DL3018 | ||
# this one just needs to be set, but since `FROM ${BASE_IMAGE}` cannot be parse, | ||
# there isn't a way to leverage this feature anyway | ||
allowedRegistries: [] |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
Copyright 2020 Authors of Cilium. All rights reserved. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# Copyright 2020 Authors of Cilium | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
REGISTRY_PREFIX ?= docker.io/cilium | ||
PUSH ?= false | ||
|
||
OUTPUT := "type=docker" | ||
ifeq ($(PUSH),true) | ||
OUTPUT := "type=registry,push=true" | ||
endif | ||
|
||
all-images: lint maker-image update-maker-image compilers-image update-compilers-image bpftool-image iproute2-image llvm-image | ||
|
||
lint: | ||
scripts/lint.sh | ||
|
||
.buildx_builder: | ||
mkdir -p .buildx | ||
docker buildx create --platform linux/amd64,linux/arm64 > $@ | ||
|
||
maker-image: .buildx_builder | ||
scripts/build-image.sh $(REGISTRY_PREFIX)/image-maker images/maker linux/amd64 $(OUTPUT) "$$(cat .buildx_builder)" | ||
|
||
update-maker-image: | ||
scripts/update-maker-image.sh $(REGISTRY_PREFIX) | ||
|
||
compilers-image: .buildx_builder | ||
scripts/build-image.sh $(REGISTRY_PREFIX)/image-compilers images/compilers linux/amd64 $(OUTPUT) "$$(cat .buildx_builder)" | ||
|
||
update-compilers-image: | ||
scripts/update-compilers-image.sh $(REGISTRY_PREFIX) | ||
|
||
bpftool-image: .buildx_builder | ||
scripts/build-image.sh $(REGISTRY_PREFIX)/bpftool images/bpftool linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)" | ||
|
||
iproute2-image: .buildx_builder | ||
scripts/build-image.sh $(REGISTRY_PREFIX)/iproute2 images/iproute2 linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)" | ||
|
||
llvm-image: .buildx_builder | ||
scripts/build-image.sh $(REGISTRY_PREFIX)/llvm images/llvm linux/amd64,linux/arm64 $(OUTPUT) "$$(cat .buildx_builder)" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
# Cilium Dependency Packaging | ||
|
||
This repository contains build definitions for a number of images that are components of the official and development images of Cilium. | ||
|
||
The builds are currently hosted in GitHub Actions, but can be ported to any other container-based CI system. | ||
|
||
## Images | ||
|
||
### [`images/maker`](images/maker/Dockerfile) | ||
|
||
This image consists of core tools used for building all other images, which include `bash`, `make` and `docker` (with [`buildx`](https://github.com/docker/buildx)) | ||
and [`crane`](https://github.com/google/go-containerregistry/blob/master/cmd/crane). | ||
This image enables using latest BuildKit features without depending on whatever Docker daemon/client CI host provides. | ||
Since `buildx` runs a BuildKit daemon inside a container, it's largely independent of what version of Docker daemon it runs on. | ||
|
||
This image also includes a secure credentials helper - [`docker-credential-env`](http://github.com/errordeveloper/docker-credential-env), | ||
which prevents having to use `docker login` which stores a plain text token in `${DOCKER_CONFIG}/config.json`. | ||
|
||
### [`images/compiler`](images/compilers/Dockerfile) | ||
|
||
This image consists of compilers and libraries needed to build other images for `amd64` and `arm64`. | ||
|
||
### [`images/bpftool`](images/bpftool/Dockerfile) | ||
|
||
This image builds `bpftool` binary for `amd64` and `arm64` using a cross-compiler. The resulting image has only one file - | ||
`/bin/bpftool`, it is a proper multi-platform image. The binary is dynamically linked to Ubuntu 20.04 glibc and other dependencies. | ||
|
||
This image is uses a recent version of `bpftool` from `bpf-next` Linux kernel tree. | ||
|
||
### [`images/iproute2`](images/iproute2/Dockerfile) | ||
|
||
This image builds `ip` and `tc` binaries for `amd64` and `arm64` using a cross-compiler. The resulting image has only two files - | ||
`/bin/ip` and `/bin/tc`, it is a proper multi-platform image. The binaries are dynamically linked to Ubuntu 20.04 glibc and other | ||
dependencies. | ||
|
||
This image is uses [a fork of `iproute2`](https://github.com/cilium/iproute2), it has features that Cilium relies on. | ||
|
||
### [`images/llvm`](images/llvm/Dockerfile) | ||
|
||
This image builds `llc` and `clang` binaries for `amd64` and `arm64` using a cross-compiler. The resulting image has only two | ||
files - `/bin/llc` and `/bin/clang`, it is a proper multi-platform image. The binaries are dynamically linked to Ubuntu 20.04 glibc | ||
and other dependencies. | ||
|
||
This image is a custom BPF-only distribution of LLVM. | ||
|
||
## Usage | ||
|
||
### Making changes | ||
|
||
All images get automatic tags based on checked-in contents of image subdirectory. At any point in git history of a subdirectory | ||
there exists a unique [git tree object hash](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), that is what's used for | ||
image tags. | ||
|
||
As the result of this, following stands: | ||
|
||
- image build definitions can be obtained with `git show <tag>` | ||
- image build is defined by contents of a directory | ||
- when changes are committed to image directory, new tag is generated | ||
- if there is a new tag, image is rebuilt and pushed with that new tag | ||
|
||
This does not cater for reproducible builds, however it serves as basis for reliable builds, especially when following rules | ||
are also applied to any build definitions: | ||
|
||
- all `FROM` statements use digests (use `scripts/get-image-digest.sh`) | ||
- any system packages are installed in a separate image that is references by a digests (that's how `images/compilers` is designed) | ||
- pining system packages can be quite laborious, especially because most of the time what you want is latest that the distribution offers, | ||
so what's much easier to let the package manager get the latest and then pin down the result by digest, so every time there is a change | ||
in underlying system packages, that is explicitly recorded by change of digest in each image that uses the base image | ||
|
||
Be sure to use `make lint`, which will run [`shellcheck`](https://github.com/koalaman/shellcheck) and [`hadolint`](https://github.com/hadolint/hadolint). | ||
|
||
For details of how this works, see the following: | ||
|
||
- [`Makefile`](Makefile) | ||
- [`scripts/build-image.sh`](`scripts/build-image.sh`) | ||
- [`scripts/make-image-tag.sh`](scripts/make-image-tag.sh). | ||
- [`images/maker`](images/maker/Dockerfile) | ||
- [`images/compilers`](images/compilers/Dockerfile) | ||
|
||
### Building Locally | ||
|
||
One should be able to build images locally as long as they have Docker installed with [`buildx` plug-in](https://docs.docker.com/buildx/working-with-buildx/). | ||
|
||
#### `images/{maker,compilers}` | ||
|
||
When you have dependencies that need to be added to these image before using them in one of the other images, e.g. if you need to add a system | ||
library in `compilers` image that will be used for compiling something else, you should make a PR to update `compilers` first. | ||
However, that's only required for full integration, and you can build images locally if you prefer, you can also push them to your own Docker Hub | ||
account or whatever is your preferred registry. | ||
|
||
When changes to these images are merged into master, builds should run and push new images to each of the registries. | ||
Once new images are out, a PR will be required to update all dependent images, please use the following commands to | ||
make updates and commit the resulting changes: | ||
|
||
> NOTE: You can only use the `update-*-image` make targets when you are _not building localy_. For example, if you have built a new `compilers` | ||
> locally, or perahps even pushed it your Docker Hyb account, and you want to consume this new version to build new `llvm` image - you need to uptade | ||
> `images/llvm/Dockerfile` manually, as the `scripts/update-*-images.sh` is not capable of handling this. | ||
- `make update-maker-image` | ||
- `make update-compilers-image` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
Dockerfile |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
# Copyright 2020 Authors of Cilium | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
ARG COMPILERS_IMAGE=docker.io/cilium/image-compilers:818f48aa3eada1d23018f02f3c43a169c51eea1a@sha256:54e7c0d8713f479e578dbcfd915795db9b12761d6397bf7ef61ae4b1d2031f03 | ||
|
||
FROM --platform=linux/amd64 ${COMPILERS_IMAGE} as builder | ||
|
||
COPY checkout-linux.sh /tmp/checkout-linux.sh | ||
RUN /tmp/checkout-linux.sh | ||
|
||
COPY build-bpftool-native.sh /tmp/build-bpftool-native.sh | ||
RUN /tmp/build-bpftool-native.sh | ||
|
||
COPY build-bpftool-cross-aarch64.sh /tmp/build-bpftool-cross-aarch64.sh | ||
RUN /tmp/build-bpftool-cross-aarch64.sh | ||
|
||
FROM scratch | ||
LABEL maintainer="[email protected]" | ||
ARG TARGETPLATFORM | ||
COPY --from=builder /out/${TARGETPLATFORM}/bin /bin |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#!/bin/bash | ||
|
||
# Copyright 2017-2020 Authors of Cilium | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
set -o xtrace | ||
set -o errexit | ||
set -o pipefail | ||
set -o nounset | ||
|
||
triplet="aarch64-linux-gnu" | ||
|
||
cd /src/linux/tools/bpf/bpftool | ||
|
||
make clean | ||
|
||
make -j "$(getconf _NPROCESSORS_ONLN)" ARCH=arm64 CROSS_COMPILE=${triplet}- | ||
|
||
${triplet}-strip bpftool | ||
|
||
mkdir -p /out/linux/arm64/bin | ||
cp bpftool /out/linux/arm64/bin |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
|
||
# Copyright 2017-2020 Authors of Cilium | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
set -o xtrace | ||
set -o errexit | ||
set -o pipefail | ||
set -o nounset | ||
|
||
cd /src/linux/tools/bpf/bpftool | ||
|
||
make -j "$(getconf _NPROCESSORS_ONLN)" | ||
|
||
strip bpftool | ||
|
||
mkdir -p /out/linux/amd64/bin | ||
cp bpftool /out/linux/amd64/bin |
Oops, something went wrong.