From a8121e1785ac9a3d6504ec5bcd0eb55819afd51c Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Fri, 17 May 2024 01:41:16 +0300 Subject: [PATCH] [1.30] Refactor build process to enable auto-update of components (#419) * Refactor build process to enable auto-update of components (#416) * use release-1.30 branch for kubernetes and k8s-dqlite --- .../ISSUE_TEMPLATE/create_release_branch.md | 20 ++- .github/workflows/go.yaml | 1 + .github/workflows/integration.yaml | 1 + .github/workflows/python.yaml | 1 + .github/workflows/sbom.yaml | 1 + .github/workflows/strict-integration.yaml | 1 + .github/workflows/update-components.yaml | 49 ++++++++ .github/workflows/update-versions.yaml | 53 -------- build-scripts/README.md | 4 +- build-scripts/build-component.sh | 2 +- build-scripts/components/cni/version | 1 + build-scripts/components/cni/version.sh | 4 - build-scripts/components/containerd/version | 1 + .../components/containerd/version.sh | 3 - build-scripts/components/helm/version | 1 + build-scripts/components/helm/version.sh | 3 - build-scripts/components/k8s-dqlite/version | 1 + .../components/k8s-dqlite/version.sh | 3 - build-scripts/components/kubernetes/version | 1 + .../components/kubernetes/version.sh | 15 --- build-scripts/components/runc/version | 1 + build-scripts/components/runc/version.sh | 5 - build-scripts/generate-bom.py | 2 +- build-scripts/hack/generate-sbom.py | 107 ++++++---------- .../hack/update-component-versions.py | 116 ++++++++++++++++++ build-scripts/hack/util.py | 50 ++++++++ .../patches/strict/0001-Strict-patch.patch | 32 ++--- build-scripts/print-patches-for.py | 16 ++- snap/snapcraft.yaml | 24 ++-- 29 files changed, 315 insertions(+), 204 deletions(-) create mode 100644 .github/workflows/update-components.yaml delete mode 100644 .github/workflows/update-versions.yaml create mode 100644 build-scripts/components/cni/version delete mode 100755 build-scripts/components/cni/version.sh create mode 100644 build-scripts/components/containerd/version delete mode 100755 build-scripts/components/containerd/version.sh create mode 100644 build-scripts/components/helm/version delete mode 100755 build-scripts/components/helm/version.sh create mode 100755 build-scripts/components/k8s-dqlite/version delete mode 100755 build-scripts/components/k8s-dqlite/version.sh create mode 100644 build-scripts/components/kubernetes/version delete mode 100755 build-scripts/components/kubernetes/version.sh create mode 100644 build-scripts/components/runc/version delete mode 100755 build-scripts/components/runc/version.sh create mode 100755 build-scripts/hack/update-component-versions.py create mode 100644 build-scripts/hack/util.py diff --git a/.github/ISSUE_TEMPLATE/create_release_branch.md b/.github/ISSUE_TEMPLATE/create_release_branch.md index b7386518b..dcf352025 100644 --- a/.github/ISSUE_TEMPLATE/create_release_branch.md +++ b/.github/ISSUE_TEMPLATE/create_release_branch.md @@ -20,6 +20,9 @@ Make sure to follow the steps below and ensure all actions are completed and sig - **PR**: +- + +- **PR**: #### Actions @@ -94,12 +97,16 @@ The steps are to be followed in-order, each task must be completed by the person - `rm -rf ~/tmp/release-1.xx` - [ ] **Reviewer**: Ensure `release-1.xx` branch is based on latest changes on `main` at the time of the release cut. - [ ] **Owner**: Create PR to initialize `release-1.xx` branch: - - [ ] Update `KUBE_TRACK` to `1.xx` in [/build-scripts/components/kubernetes/version.sh][] - - [ ] Update `master` to `release-1.xx` in [/build-scripts/components/k8s-dqlite/version.sh][] + - [ ] Update `KUBERNETES_RELEASE_MARKER` to `stable-1.xx` in [/build-scripts/hack/update-component-versions.py][] + - [ ] Update `master` to `release-1.xx` in [/build-scripts/components/k8s-dqlite/version][] - [ ] Update `"main"` to `"release-1.xx"` in [/build-scripts/hack/generate-sbom.py][] - [ ] `git commit -m 'Release 1.xx'` - - [ ] Create PR with the changes and request review from **Reviewer**. Make sure to update the issue `Information` section with a link to the PR. + - [ ] Create PR against `release-1.xx` with the changes and request review from **Reviewer**. Make sure to update the issue `Information` section with a link to the PR. - [ ] **Reviewer**: Review and merge PR to initialize branch. +- [ ] **Owner**: Create PR to initialize `update-components.yaml` job for `release-1.xx` branch: + - [ ] Add `release-1.xx` in [.github/workflows/update-components.yaml][] + - [ ] Remove unsupported releases from the list (if applicable, consult with **Reviewer**) + - [ ] Create PR against `main` with the changes and request review from **Reviewer**. Make sure to update the issue information with a link to the PR. - [ ] **Reviewer**: On merge, confirm [Auto-update strict branch] action runs to completion and that the `autoupdate/release-1.xx-strict` branch is created. - [ ] **Owner**: Create launchpad builders for `release-1.xx` - [ ] Go to [lp:k8s][] and do **Import now** to pick up all latest changes. @@ -148,9 +155,10 @@ The steps are to be followed in-order, each task must be completed by the person [.github/workflows/python.yaml]: ../workflows/python.yaml [.github/workflows/sbom.yaml]: ../workflows/sbom.yaml [.github/workflows/strict-integration.yaml]: ../workflows/strict-integration.yaml -[.github/workflows/strict.yaml]: ..workflows/strict.yaml -[/build-scripts/components/kubernetes/version.sh]: ../../build-scripts/components/kubernetes/version.sh -[/build-scripts/components/k8s-dqlite/version.sh]: ../../build-scripts/components/k8s-dqlite/version.sh +[.github/workflows/strict.yaml]: ../workflows/strict.yaml +[.github/workflows/update-components.yaml]: ../workflows/update-components.yaml +[/build-scripts/components/hack/update-component-versions.py]: ../../build-scripts/components/hack/update-component-versions.py +[/build-scripts/components/k8s-dqlite/version]: ../../build-scripts/components/k8s-dqlite/version [/build-scripts/hack/generate-sbom.py]: ../..//build-scripts/hack/generate-sbom.py [lp:k8s]: https://code.launchpad.net/~cdk8s/k8s/+git/k8s-snap [lp:k8s/+snaps]: https://launchpad.net/k8s/+snaps diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index aacb9d6e6..e800c5838 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -7,6 +7,7 @@ on: - autoupdate/strict - 'release-[0-9]+.[0-9]+' - 'autoupdate/release-[0-9]+.[0-9]+-strict' + - 'autoupdate/sync/+' pull_request: jobs: diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index d49790df4..56c1ee8c7 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -7,6 +7,7 @@ on: - autoupdate/strict - 'release-[0-9]+.[0-9]+' - 'autoupdate/release-[0-9]+.[0-9]+-strict' + - 'autoupdate/sync/+' pull_request: jobs: diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 5d0935328..06981daea 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -7,6 +7,7 @@ on: - autoupdate/strict - 'release-[0-9]+.[0-9]+' - 'autoupdate/release-[0-9]+.[0-9]+-strict' + - 'autoupdate/sync/+' pull_request: jobs: diff --git a/.github/workflows/sbom.yaml b/.github/workflows/sbom.yaml index 4a39c8369..13e47ff6b 100644 --- a/.github/workflows/sbom.yaml +++ b/.github/workflows/sbom.yaml @@ -7,6 +7,7 @@ on: - autoupdate/strict - 'release-[0-9]+.[0-9]+' - 'autoupdate/release-[0-9]+.[0-9]+-strict' + - 'autoupdate/sync/+' pull_request: jobs: diff --git a/.github/workflows/strict-integration.yaml b/.github/workflows/strict-integration.yaml index 54e349015..1b92ea576 100644 --- a/.github/workflows/strict-integration.yaml +++ b/.github/workflows/strict-integration.yaml @@ -5,6 +5,7 @@ on: branches: - main - 'release-[0-9]+.[0-9]+' + - 'autoupdate/sync/+' pull_request: jobs: diff --git a/.github/workflows/update-components.yaml b/.github/workflows/update-components.yaml new file mode 100644 index 000000000..7c5c68f11 --- /dev/null +++ b/.github/workflows/update-components.yaml @@ -0,0 +1,49 @@ +name: Check for component upgrades + +on: + workflow_dispatch: + schedule: + - cron: "0 10 * * *" + +jobs: + update: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + branch: + # Keep main branch up to date + - main + # Supported stable release branches + - release-1.30 + + steps: + - name: Checking out repo + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} + ssh-key: ${{ secrets.DEPLOY_KEY_TO_UPDATE_STRICT_BRANCH }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip3 install -r ./build-scripts/hack/requirements.txt + + - name: Check for new component versions + run: | + ./build-scripts/hack/update-component-versions.py + + - name: Create pull request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.DEPLOY_KEY_TO_UPDATE_STRICT_BRANCH }} + commit-message: "[${{ matrix.branch }}] Update component versions" + title: "[${{ matrix.branch }}] Update component versions" + body: "[${{ matrix.branch }}] Update component versions" + branch: "autoupdate/sync/${{ matrix.branch }}" + delete-branch: true + base: ${{ matrix.branch }} diff --git a/.github/workflows/update-versions.yaml b/.github/workflows/update-versions.yaml deleted file mode 100644 index 634ee4d80..000000000 --- a/.github/workflows/update-versions.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: Create PRs for new versions of parts - -on: - workflow_dispatch: - schedule: - - cron: "0 10 * * *" - -jobs: - update-versions: - runs-on: ubuntu-latest - strategy: - matrix: - part: ["cni", "containerd", "helm", "runc"] - - steps: - - name: Checking out repo - uses: actions/checkout@v4 - with: - ref: main - - - name: Fetch and filter upstream tags - id: fetch-upstream - run: | - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update - sudo apt-get -qq install -y jq - - repository="$(cat "build-scripts/components/${{ matrix.part }}/repository")" - current_tag="$("build-scripts/components/${{ matrix.part }}/version.sh")" - - IFS='.' read -r -a current_tag_parts <<< "$current_tag" - - git clone $repository part - - # Get the latest tag for major version, excluding -rc etc. - latest_tag="$(git -C part tag --sort=-v:refname | grep "${current_tag_parts[0]}" | grep -vE "-" | head -n 1)" - - rm -rf part - - if dpkg --compare-versions "${latest_tag:1}" "gt" "${current_tag:1}" ; then - echo "Found newer tag $latest_tag" - echo "new-tag=$latest_tag" >> $GITHUB_OUTPUT - sed -i "/echo/c\\echo \"$latest_tag\"" "build-scripts/components/${{ matrix.part }}/version.sh" - fi - - - name: Create pull request - uses: peter-evans/create-pull-request@v6 - with: - commit-message: Update ${{ matrix.part }} version to ${{ steps.fetch-upstream.outputs.new-tag }} - title: "[${{ matrix.part }}] Update version to ${{ steps.fetch-upstream.outputs.new-tag }}" - body: Update ${{ matrix.part }} version to ${{ steps.fetch-upstream.outputs.new-tag }} - branch: update/${{ matrix.part }}-${{ steps.fetch-upstream.outputs.new-tag }} - base: main diff --git a/build-scripts/README.md b/build-scripts/README.md index c32bd5e7f..fdaaf6394 100644 --- a/build-scripts/README.md +++ b/build-scripts/README.md @@ -14,7 +14,7 @@ build-scripts/ components/ $component_name/ repository <-- git repository to clone - version.sh <-- prints the repository tag or commit to checkout + version <-- repository tag or commit to checkout build.sh <-- runs as `build.sh $output $version` first argument is the output directory where binaries should be placed, second is the component version @@ -26,7 +26,7 @@ build-scripts/ ## Applying patches -Most K8s components are retrieved from an upstream source (specified in the `repository`), with a specific tag (specified in `version.sh`), have some patches applied to them (from the `patches/` directory) and are then built (using `build.sh`). +Most K8s components are retrieved from an upstream source (specified in the `repository`), with a specific tag (specified in `version`), have some patches applied to them (from the `patches/` directory) and are then built (using `build.sh`). This section explains the directory format for the `patches` directory. diff --git a/build-scripts/build-component.sh b/build-scripts/build-component.sh index 149250eba..bb38ae54b 100755 --- a/build-scripts/build-component.sh +++ b/build-scripts/build-component.sh @@ -13,7 +13,7 @@ COMPONENT_NAME="${1}" COMPONENT_DIRECTORY="${DIR}/components/${COMPONENT_NAME}" GIT_REPOSITORY="$(cat "${COMPONENT_DIRECTORY}/repository")" -GIT_TAG="$("${COMPONENT_DIRECTORY}/version.sh")" +GIT_TAG="$(cat "${COMPONENT_DIRECTORY}/version")" COMPONENT_BUILD_DIRECTORY="${BUILD_DIRECTORY}/${COMPONENT_NAME}" diff --git a/build-scripts/components/cni/version b/build-scripts/components/cni/version new file mode 100644 index 000000000..0d0c52f84 --- /dev/null +++ b/build-scripts/components/cni/version @@ -0,0 +1 @@ +v1.4.0 diff --git a/build-scripts/components/cni/version.sh b/build-scripts/components/cni/version.sh deleted file mode 100755 index 3afd859f7..000000000 --- a/build-scripts/components/cni/version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# Match https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml#L20 -echo "v1.3.0" diff --git a/build-scripts/components/containerd/version b/build-scripts/components/containerd/version new file mode 100644 index 000000000..4b45cf7ee --- /dev/null +++ b/build-scripts/components/containerd/version @@ -0,0 +1 @@ +v1.6.28 diff --git a/build-scripts/components/containerd/version.sh b/build-scripts/components/containerd/version.sh deleted file mode 100755 index 151d227ac..000000000 --- a/build-scripts/components/containerd/version.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "v1.6.28" diff --git a/build-scripts/components/helm/version b/build-scripts/components/helm/version new file mode 100644 index 000000000..62b6d193d --- /dev/null +++ b/build-scripts/components/helm/version @@ -0,0 +1 @@ +v3.14.2 diff --git a/build-scripts/components/helm/version.sh b/build-scripts/components/helm/version.sh deleted file mode 100755 index f54c43b3e..000000000 --- a/build-scripts/components/helm/version.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "v3.14.2" diff --git a/build-scripts/components/k8s-dqlite/version b/build-scripts/components/k8s-dqlite/version new file mode 100755 index 000000000..d6a729989 --- /dev/null +++ b/build-scripts/components/k8s-dqlite/version @@ -0,0 +1 @@ +release-1.30 diff --git a/build-scripts/components/k8s-dqlite/version.sh b/build-scripts/components/k8s-dqlite/version.sh deleted file mode 100755 index b20629c9f..000000000 --- a/build-scripts/components/k8s-dqlite/version.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "release-1.30" diff --git a/build-scripts/components/kubernetes/version b/build-scripts/components/kubernetes/version new file mode 100644 index 000000000..c8c2eaea5 --- /dev/null +++ b/build-scripts/components/kubernetes/version @@ -0,0 +1 @@ +v1.30.0 diff --git a/build-scripts/components/kubernetes/version.sh b/build-scripts/components/kubernetes/version.sh deleted file mode 100755 index 3165e1465..000000000 --- a/build-scripts/components/kubernetes/version.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -KUBE_TRACK="1.30" # example: "1.24" -KUBE_VERSION="${KUBE_VERSION:-}" # example: "v1.24.2" - -if [ -z "${KUBE_VERSION}" ]; then - if [ -z "${KUBE_TRACK}" ]; then - KUBE_VERSION="$(curl -L --silent "https://dl.k8s.io/release/stable.txt")" - else - KUBE_TRACK="${KUBE_TRACK#v}" - KUBE_VERSION="$(curl -L --silent "https://dl.k8s.io/release/stable-${KUBE_TRACK}.txt")" - fi -fi - -echo "${KUBE_VERSION}" diff --git a/build-scripts/components/runc/version b/build-scripts/components/runc/version new file mode 100644 index 000000000..19f5e1b57 --- /dev/null +++ b/build-scripts/components/runc/version @@ -0,0 +1 @@ +v1.1.12 diff --git a/build-scripts/components/runc/version.sh b/build-scripts/components/runc/version.sh deleted file mode 100755 index f0e7bc9b1..000000000 --- a/build-scripts/components/runc/version.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# Containerd version is defined in ../containerd/version.sh -# Match with version of github.com/opencontainers/runc dependency in https://github.com/containerd/containerd/blob/$containerd_version/go.mod -echo "v1.1.12" diff --git a/build-scripts/generate-bom.py b/build-scripts/generate-bom.py index 3d42179ee..a8326dbec 100755 --- a/build-scripts/generate-bom.py +++ b/build-scripts/generate-bom.py @@ -51,7 +51,7 @@ def _read_file(path: Path) -> str: component_dir = DIR / "components" / component try: - version = _parse_output([component_dir / "version.sh"]) + version = _read_file(component_dir / "version") patches = _parse_output([sys.executable, DIR / "print-patches-for.py", component, version]) clean_patches = [] if patches: diff --git a/build-scripts/hack/generate-sbom.py b/build-scripts/hack/generate-sbom.py index 9d1153775..78aeefa7f 100755 --- a/build-scripts/hack/generate-sbom.py +++ b/build-scripts/hack/generate-sbom.py @@ -10,16 +10,14 @@ """ import argparse -import contextlib import json import logging -import subprocess import sys import tarfile import tempfile import yaml from pathlib import Path -from typing import Any, Generator +import util logging.basicConfig(level=logging.INFO) @@ -27,7 +25,7 @@ DIR = Path(__file__).absolute().parent -SNAPCRAFT_YAML = yaml.safe_load(Path(DIR / "../../snap/snapcraft.yaml").read_text()) +SNAPCRAFT_YAML = yaml.safe_load(util.read_file(DIR / "../../snap/snapcraft.yaml")) # FIXME: This information should not be hardcoded here CILIUM_ROCK_REPO = "https://github.com/canonical/cilium-rocks" @@ -50,37 +48,6 @@ K8S_DIR = DIR / "../../src/k8s" -@contextlib.contextmanager -def _git_repo(repo_url: str, repo_tag: str) -> Generator[Path, Any, Any]: - """ - Clone a git repository on a temporary directory and return the directory. - - Example usage: - - ``` - with _git_repo("https://github.com/canonical/k8s-snap", "master") as dir: - print("Repo cloned at", dir) - ``` - - """ - with tempfile.TemporaryDirectory() as tmpdir: - LOG.info("Cloning %s @ %s", repo_url, repo_tag) - _parse_output(["git", "clone", repo_url, tmpdir, "-b", repo_tag, "--depth=1"]) - yield Path(tmpdir) - - -def _parse_output(*args, **kwargs): - return ( - subprocess.run(*args, capture_output=True, check=True, **kwargs) - .stdout.decode() - .strip() - ) - - -def _read_file(path: Path) -> str: - return path.read_text().strip() - - def c_components_from_snapcraft(manifest, extra_files): for component in SNAPCRAFT_C_COMPONENTS: LOG.info("Generating SBOM info for C component %s", component) @@ -96,16 +63,16 @@ def c_components_from_snapcraft(manifest, extra_files): def go_components_external(manifest, extra_files): for component in SNAPCRAFT_GO_COMPONENTS: LOG.info("Generating SBOM info for Go component %s", component) - repo_url = _read_file(DIR / "../components" / component / "repository") - repo_tag = _parse_output([DIR / "../components" / component / "version.sh"]) + repo_url = util.read_file(DIR / "../components" / component / "repository") + repo_tag = util.read_file(DIR / "../components" / component / "version") go_mod_name = f"{component}/go.mod" go_sum_name = f"{component}/go.sum" - with _git_repo(repo_url, repo_tag) as dir: - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) - extra_files[go_mod_name] = _read_file(Path(dir) / "go.mod") - extra_files[go_sum_name] = _read_file(Path(dir) / "go.sum") + with util.git_repo(repo_url, repo_tag) as dir: + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + extra_files[go_mod_name] = util.read_file(Path(dir) / "go.mod") + extra_files[go_sum_name] = util.read_file(Path(dir) / "go.sum") manifest["snap"]["external"][component] = { "language": "go", @@ -121,16 +88,16 @@ def go_components_external(manifest, extra_files): def k8s_snap_go_components(manifest, extra_files): LOG.info("Generating SBOM info for k8s-snap") - extra_files["k8s-snap/go.mod"] = _read_file(K8S_DIR / "go.mod") - extra_files["k8s-snap/go.sum"] = _read_file(K8S_DIR / "go.sum") + extra_files["k8s-snap/go.mod"] = util.read_file(K8S_DIR / "go.mod") + extra_files["k8s-snap/go.sum"] = util.read_file(K8S_DIR / "go.sum") manifest["snap"]["k8s-snap"]["k8s-snap"] = { "language": "go", "details": ["k8s-snap/go.mod", "k8s-snap/go.sum"], "source": { "type": "git", - "repo": _parse_output(["git", "remote", "get-url", "origin"]), - "tag": _parse_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]), - "revision": _parse_output(["git", "rev-parse", "HEAD"]), + "repo": util.parse_output(["git", "remote", "get-url", "origin"]), + "tag": util.parse_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]), + "revision": util.parse_output(["git", "rev-parse", "HEAD"]), }, } @@ -155,8 +122,8 @@ def k8s_snap_c_dqlite_components(manifest, extra_files): for component in repos: repo_url = repos[component] repo_tag = tags[component] - with _git_repo(repo_url, repo_tag) as dir: - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + with util.git_repo(repo_url, repo_tag) as dir: + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) manifest["snap"]["k8s-snap"][component] = { "language": "c", @@ -172,8 +139,8 @@ def k8s_snap_c_dqlite_components(manifest, extra_files): def rock_cilium(manifest, extra_files): LOG.info("Generating SBOM info for Cilium rocks") - with _git_repo(CILIUM_ROCK_REPO, CILIUM_ROCK_TAG) as d: - rock_repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=d) + with util.git_repo(CILIUM_ROCK_REPO, CILIUM_ROCK_TAG) as d: + rock_repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=d) rockcraft = (d / "cilium/rockcraft.yaml").read_text() operator_rockcraft = (d / "cilium-operator-generic/rockcraft.yaml").read_text() @@ -184,13 +151,13 @@ def rock_cilium(manifest, extra_files): repo_url = rockcraft_yaml["parts"]["cilium"]["source"] repo_tag = rockcraft_yaml["parts"]["cilium"]["source-tag"] - with _git_repo(repo_url, repo_tag) as dir: - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) - extra_files["cilium/go.mod"] = _read_file(dir / "go.mod") - extra_files["cilium/go.sum"] = _read_file(dir / "go.sum") + with util.git_repo(repo_url, repo_tag) as dir: + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + extra_files["cilium/go.mod"] = util.read_file(dir / "go.mod") + extra_files["cilium/go.sum"] = util.read_file(dir / "go.sum") - extra_files["cilium-operator-generic/go.mod"] = _read_file(dir / "go.mod") - extra_files["cilium-operator-generic/go.sum"] = _read_file(dir / "go.sum") + extra_files["cilium-operator-generic/go.mod"] = util.read_file(dir / "go.mod") + extra_files["cilium-operator-generic/go.sum"] = util.read_file(dir / "go.sum") # NOTE: this silently assumes that cilium and cilium-operator-generic rocks are in sync manifest["rocks"]["cilium"] = { @@ -221,8 +188,8 @@ def rock_cilium(manifest, extra_files): def rock_coredns(manifest, extra_files): LOG.info("Generating SBOM info for CoreDNS rock") - with _git_repo(COREDNS_ROCK_REPO, COREDNS_ROCK_TAG) as d: - rock_repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=d) + with util.git_repo(COREDNS_ROCK_REPO, COREDNS_ROCK_TAG) as d: + rock_repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=d) rockcraft = (d / "rockcraft.yaml").read_text() extra_files["coredns/rockcraft.yaml"] = rockcraft @@ -231,10 +198,10 @@ def rock_coredns(manifest, extra_files): repo_url = rockcraft_yaml["parts"]["coredns"]["source"] repo_tag = rockcraft_yaml["parts"]["coredns"]["source-tag"] - with _git_repo(repo_url, repo_tag) as dir: - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) - extra_files["coredns/go.mod"] = _read_file(dir / "go.mod") - extra_files["coredns/go.sum"] = _read_file(dir / "go.sum") + with util.git_repo(repo_url, repo_tag) as dir: + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + extra_files["coredns/go.mod"] = util.read_file(dir / "go.mod") + extra_files["coredns/go.sum"] = util.read_file(dir / "go.sum") manifest["rocks"]["coredns"] = { "rock-source": { @@ -257,8 +224,8 @@ def rock_coredns(manifest, extra_files): def rock_metrics_server(manifest, extra_files): LOG.info("Generating SBOM info for metrics-server rock") - with _git_repo(METRICS_SERVER_ROCK_REPO, METRICS_SERVER_ROCK_TAG) as d: - rock_repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=d) + with util.git_repo(METRICS_SERVER_ROCK_REPO, METRICS_SERVER_ROCK_TAG) as d: + rock_repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=d) rockcraft = (d / "rockcraft.yaml").read_text() extra_files["metrics-server/rockcraft.yaml"] = rockcraft @@ -267,10 +234,10 @@ def rock_metrics_server(manifest, extra_files): repo_url = rockcraft_yaml["parts"]["metrics-server"]["source"] repo_tag = rockcraft_yaml["parts"]["metrics-server"]["source-tag"] - with _git_repo(repo_url, repo_tag) as dir: - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) - extra_files["metrics-server/go.mod"] = _read_file(dir / "go.mod") - extra_files["metrics-server/go.sum"] = _read_file(dir / "go.sum") + with util.git_repo(repo_url, repo_tag) as dir: + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + extra_files["metrics-server/go.mod"] = util.read_file(dir / "go.mod") + extra_files["metrics-server/go.sum"] = util.read_file(dir / "go.sum") manifest["rocks"]["metrics-server"] = { "rock-source": { @@ -300,11 +267,11 @@ def rock_rawfile_localpv(manifest, extra_files): repo_url = RAWFILE_LOCALPV_REPO repo_tag = RAWFILE_LOCALPV_TAG - with _git_repo(repo_url, repo_tag) as dir: + with util.git_repo(repo_url, repo_tag) as dir: rockcraft = (dir / "rockcraft.yaml").read_text() requirements = (dir / "requirements.txt").read_text() - repo_commit = _parse_output(["git", "rev-parse", "HEAD"], cwd=dir) + repo_commit = util.parse_output(["git", "rev-parse", "HEAD"], cwd=dir) extra_files["rawfile-localpv/rockcraft.yaml"] = rockcraft extra_files["rawfile-localpv/requirements.txt"] = requirements diff --git a/build-scripts/hack/update-component-versions.py b/build-scripts/hack/update-component-versions.py new file mode 100755 index 000000000..7464c94e7 --- /dev/null +++ b/build-scripts/hack/update-component-versions.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +USAGE = "Update component versions for Canonical Kubernetes" + +DESCRIPTION = """ +Update the component versions that we use to build Canonical Kubernetes. This +script updates the individual `build-scripts/components//version` +files. The logic for each component is different, and is managed by configuration +options found in this script. +""" + + +import argparse +import logging +import sys +import yaml +from pathlib import Path +import util + +logging.basicConfig(level=logging.INFO) + +LOG = logging.getLogger(__name__) + +DIR = Path(__file__).absolute().parent +COMPONENTS = DIR.parent / "components" + +# Version marker for latest Kubernetes version. Expected to be one of: +# +# - "https://dl.k8s.io/release/stable.txt" +# - "https://dl.k8s.io/release/stable-1.xx.txt" +KUBERNETES_VERSION_MARKER = "https://dl.k8s.io/release/stable-1.30.txt" + +# Containerd release branch to track. The most recent tag in the branch will be used. +CONTAINERD_RELEASE_BRANCH = "release/1.6" + +# Helm release branch to track. The most recent tag in the branch will be used. +HELM_RELEASE_BRANCH = "release-3.14" + + +def get_kubernetes_version() -> str: + """Update Kubernetes version based on the specified marker file""" + LOG.info("Checking latest Kubernetes version from %s", KUBERNETES_VERSION_MARKER) + return util.read_url(KUBERNETES_VERSION_MARKER) + + +def get_cni_version() -> str: + """Update CNI version to match the CNI version used in $kubernetes/build/dependencies.yaml""" + kube_repo = util.read_file(COMPONENTS / "kubernetes/repository") + kube_version = util.read_file(COMPONENTS / "kubernetes/version") + + with util.git_repo(kube_repo, kube_version) as dir: + deps_file = dir / "build/dependencies.yaml" + deps = yaml.safe_load(util.read_file(deps_file)) + + for dep in deps["dependencies"]: + if dep["name"] == "cni": + ersion = dep["version"] + return f"v{ersion.lstrip('v')}" + + raise Exception(f"Failed to find cni dependency in {deps_file}") + + +def get_containerd_version() -> str: + """Update containerd version using latest tag of specified branch""" + containerd_repo = util.read_file(COMPONENTS / "containerd/repository") + + with util.git_repo(containerd_repo, CONTAINERD_RELEASE_BRANCH, shallow=False) as dir: + # Get the latest tagged release from the current branch + return util.parse_output(["git", "describe", "--tags", "--abbrev=0"], cwd=dir) + + +def get_runc_version() -> str: + """Update runc version based on containerd""" + containerd_repo = util.read_file(COMPONENTS / "containerd/repository") + containerd_version = util.read_file(COMPONENTS / "containerd/version") + + with util.git_repo(containerd_repo, containerd_version) as dir: + # See https://github.com/containerd/containerd/blob/main/docs/RUNC.md + return util.read_file(dir / "script/setup/runc-version") + + +def get_helm_version() -> str: + """Get latest version of helm""" + helm_repo = util.read_file(COMPONENTS / "helm/repository") + with util.git_repo(helm_repo, HELM_RELEASE_BRANCH, shallow=False) as dir: + return util.parse_output(["git", "describe", "--tags", "--abbrev=0"], cwd=dir) + + +def update_component_versions(dry_run: bool): + for component, get_version in [ + ("kubernetes", get_kubernetes_version), + ("cni", get_cni_version), + ("containerd", get_containerd_version), + ("runc", get_runc_version), + ("helm", get_helm_version), + ]: + LOG.info("Updating version for %s", component) + version: str = get_version() + path = COMPONENTS / component / "version" + LOG.info("Update %s version to %s in %s", component, version, path) + if not dry_run: + Path(path).write_text(version.strip() + "\n") + + +def main(): + parser = argparse.ArgumentParser( + "update-component-versions.py", usage=USAGE, description=DESCRIPTION + ) + parser.add_argument("--dry-run", default=False, action="store_true") + args = parser.parse_args(sys.argv[1:]) + + return update_component_versions(args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/build-scripts/hack/util.py b/build-scripts/hack/util.py new file mode 100644 index 000000000..871d5b47c --- /dev/null +++ b/build-scripts/hack/util.py @@ -0,0 +1,50 @@ +import contextlib +from typing import Any, Generator +import tempfile +import subprocess +import logging +from urllib.request import urlopen +from pathlib import Path + +LOG = logging.getLogger(__name__) + + +@contextlib.contextmanager +def git_repo( + repo_url: str, repo_tag: str, shallow: bool = True +) -> Generator[Path, Any, Any]: + """ + Clone a git repository on a temporary directory and return the directory. + + Example usage: + + ``` + with git_repo("https://github.com/canonical/k8s-snap", "master") as dir: + print("Repo cloned at", dir) + ``` + + """ + + with tempfile.TemporaryDirectory() as tmpdir: + cmd = ["git", "clone", repo_url, tmpdir, "-b", repo_tag] + if shallow: + cmd.extend(["--depth", "1"]) + LOG.info("Cloning %s @ %s (shallow=%s)", repo_url, repo_tag, shallow) + parse_output(cmd) + yield Path(tmpdir) + + +def parse_output(*args, **kwargs): + return ( + subprocess.run(*args, capture_output=True, check=True, **kwargs) + .stdout.decode() + .strip() + ) + + +def read_file(path: Path) -> str: + return path.read_text().strip() + + +def read_url(url: str) -> str: + return urlopen(url).read().decode().strip() diff --git a/build-scripts/patches/strict/0001-Strict-patch.patch b/build-scripts/patches/strict/0001-Strict-patch.patch index eb2728cdd..2ca7c50fa 100644 --- a/build-scripts/patches/strict/0001-Strict-patch.patch +++ b/build-scripts/patches/strict/0001-Strict-patch.patch @@ -1,30 +1,16 @@ -From 7e71f38adb094e2aa5c57a06daac37be09a0acfd Mon Sep 17 00:00:00 2001 +From 3338580f4e22b001615320c40b1c1ad95f8a945e Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis -Date: Wed, 17 Apr 2024 00:35:00 +0300 +Date: Fri, 10 May 2024 19:17:55 +0300 Subject: [PATCH] Strict patch --- - build-scripts/print-patches-for.py | 2 +- - k8s/hack/init.sh | 6 +- - k8s/wrappers/services/containerd | 5 - - snap/snapcraft.yaml | 168 ++++++++++++++++++++++++++++- - 4 files changed, 173 insertions(+), 8 deletions(-) + k8s/hack/init.sh | 6 +- + k8s/wrappers/services/containerd | 5 - + snap/snapcraft.yaml | 168 ++++++++++++++++++++++++++++++- + 3 files changed, 172 insertions(+), 7 deletions(-) -diff --git a/build-scripts/print-patches-for.py b/build-scripts/print-patches-for.py -index 2c65083..13ea57c 100755 ---- a/build-scripts/print-patches-for.py -+++ b/build-scripts/print-patches-for.py -@@ -5,7 +5,7 @@ from pathlib import Path - - DIR = Path(__file__).absolute().parent - --PATCH_DIRS = ["patches"] -+PATCH_DIRS = ["patches", "strict-patches"] - - - class Version: diff --git a/k8s/hack/init.sh b/k8s/hack/init.sh -index a0b57c7..d53b528 100755 +index a0b57c7d..d53b528a 100755 --- a/k8s/hack/init.sh +++ b/k8s/hack/init.sh @@ -1,3 +1,7 @@ @@ -37,7 +23,7 @@ index a0b57c7..d53b528 100755 +"${DIR}/connect-interfaces.sh" +"${DIR}/network-requirements.sh" diff --git a/k8s/wrappers/services/containerd b/k8s/wrappers/services/containerd -index c3f71a0..a82e1c0 100755 +index c3f71a01..a82e1c03 100755 --- a/k8s/wrappers/services/containerd +++ b/k8s/wrappers/services/containerd @@ -21,9 +21,4 @@ You can try to apply the profile manually by running: @@ -51,7 +37,7 @@ index c3f71a0..a82e1c0 100755 - k8s::common::execute_service containerd diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml -index 53347a0..339ca8a 100644 +index 54b5fc0b..01631684 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -7,7 +7,7 @@ description: |- diff --git a/build-scripts/print-patches-for.py b/build-scripts/print-patches-for.py index 2c650837c..0a7781d3b 100755 --- a/build-scripts/print-patches-for.py +++ b/build-scripts/print-patches-for.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 import argparse +import os from pathlib import Path DIR = Path(__file__).absolute().parent -PATCH_DIRS = ["patches"] +# SNAPCRAFT_PROJECT_DIR is set when building the snap. If unset, resolve based on current file path +_PROJECT_DIR = os.getenv("SNAPCRAFT_PROJECT_DIR") or "" +PROJECT_DIR = _PROJECT_DIR and Path(_PROJECT_DIR) or Path(DIR / "..") + +# Strict confinement needs extra patches +STRICT = "confinement: strict" in (PROJECT_DIR / "snap/snapcraft.yaml").read_text() class Version: @@ -71,10 +77,14 @@ def get_patches_for(component: str, version_string: str) -> list: with target 'version'. """ component_version = Version(version_string) - component_dir = DIR / "components" / component + component_dir = PROJECT_DIR / "build-scripts/components" / component + + patch_dirs = ["patches"] + if STRICT: + patch_dirs += ["strict-patches"] patches = [] - for patch_dir_name in PATCH_DIRS: + for patch_dir_name in patch_dirs: patches_dir = component_dir / patch_dir_name if not patches_dir.is_dir(): continue diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index e0410bb2f..54b5fc0b5 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -65,8 +65,8 @@ parts: k8s-dqlite: after: [dqlite] plugin: nil - source: build-scripts - override-build: ./build-component.sh k8s-dqlite + source: build-scripts/components/k8s-dqlite + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh k8s-dqlite k8s-binaries: after: [dqlite] @@ -86,26 +86,26 @@ parts: cni: after: [build-deps] plugin: nil - source: build-scripts - override-build: ./build-component.sh cni + source: build-scripts/components/cni + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh cni kubernetes: after: [build-deps] plugin: nil source: build-scripts - override-build: ./build-component.sh kubernetes + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh kubernetes kubernetes-version: plugin: nil source: build-scripts/components/kubernetes - override-build: snapcraftctl set-version "$(./version.sh)" + override-build: snapcraftctl set-version "$(cat ./version)" helm: after: [build-deps] build-attributes: [no-patchelf] plugin: nil - source: build-scripts - override-build: ./build-component.sh helm + source: build-scripts/components/helm + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh helm libmnl: after: [build-deps] @@ -144,15 +144,15 @@ parts: after: [runc] plugin: nil build-attributes: [no-patchelf] - source: build-scripts - override-build: ./build-component.sh containerd + source: build-scripts/components/containerd + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh containerd runc: after: [iptables, build-deps] plugin: nil build-attributes: [no-patchelf] - source: build-scripts - override-build: ./build-component.sh runc + source: build-scripts/components/runc + override-build: $SNAPCRAFT_PROJECT_DIR/build-scripts/build-component.sh runc bash-utils: plugin: nil