From 77d40c657a09779689a15935c88219869ea115bf Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Mon, 1 Jul 2024 23:42:11 +0200 Subject: [PATCH] Introduce the NativeLink Kubernetes operator A single `kubectl apply -k` now deploys NativeLink in a self-configuring, self-healing and self-updating fashion. To achieve this we implement a two-stage depoyment to asynchronously reconciliate various parts of NativeLink Kustomizations. First, we deploy Flux Alerts that trigger Tekton Pipelines on GitRepository updates to bring required images into the cluster. Second, and technically at the same time, we start a Flux Kustomization to deploy a NativeLink Kustomization. This is similar to the previous 01_operations and 02_applicaion scripts, but now happens fully automated in the cluster and no longer requires a local Nix installation as all tag evaluations have become implementation details of the Tekton Pipelines. This commit also changes the K8s resource layout to a "best-practice" Kustomize directory layout. This further reduces code duplication and gives third parties greater flexibility and more useful reference points to build custom NativeLink setups. Includes an overhaul of the Kubernetes documentation. --- .../vocabularies/TraceMachina/accept.txt | 1 + .github/workflows/lre.yaml | 29 ++- README.md | 2 +- .../chromium-example/build_chromium_tests.sh | 0 deploy/chromium-example/kustomization.yaml | 35 ++++ deploy/dev/kustomization.yaml | 69 +++++++ deploy/kubernetes-example/kustomization.yaml | 35 ++++ deployment-examples/chromium/.gitignore | 2 - deployment-examples/chromium/01_operations.sh | 39 ---- .../chromium/02_application.sh | 30 --- .../chromium/04_delete_application.sh | 6 - deployment-examples/chromium/README.md | 91 --------- deployment-examples/kubernetes/.gitignore | 2 - .../kubernetes/01_operations.sh | 39 ---- .../kubernetes/02_application.sh | 39 ---- .../kubernetes/03_delete_application.sh | 6 - deployment-examples/kubernetes/README.md | 158 --------------- .../kubernetes/base/kustomization.yaml | 27 --- .../kubernetes/worker-lre-java.yaml | 69 ------- docs/.gitignore | 2 - docs/astro.config.mjs | 8 +- docs/scripts/md_to_mdx_aot.ts | 10 - docs/src/content/docs/guides/chromium.mdx | 120 +++++++++++ docs/src/content/docs/guides/kubernetes.mdx | 191 ++++++++++++++++++ flake.nix | 7 + kubernetes/README.md | 13 ++ .../kubernetes => kubernetes}/base/cas.yaml | 0 kubernetes/base/kustomization.yaml | 13 ++ .../base/scheduler.yaml | 0 .../base/worker.yaml | 22 +- .../gateway-routes/kustomization.yaml | 5 + .../components/gateway-routes}/routes.yaml | 0 .../example-do-not-use-in-prod-key.pem | 0 .../example-do-not-use-in-prod-rootca.crt | 0 .../insecure-certs/kustomization.yaml | 9 + .../components/operator/flux-config.yaml | 110 ++++++++++ .../components/operator/kustomization.yaml | 4 + .../base => kubernetes/configmaps}/cas.json | 0 kubernetes/configmaps/kustomization.yaml | 23 +++ .../configmaps}/scheduler.json | 0 .../configmaps}/worker.json | 0 .../overlays/chromium/kustomization.yaml | 20 ++ kubernetes/overlays/lre/kustomization.yaml | 26 +++ .../overlays/lre}/worker-lre-cc.yaml | 38 +--- native-cli/clusters/localcluster.go | 6 +- native-cli/components/cilium.go | 4 +- native-cli/components/embedded/capacitor.yaml | 46 +++++ .../components/embedded/envoy.template.yaml | 2 + .../components/embedded/kustomization.yaml | 2 + .../embedded/nativelink-gateways.yaml | 12 ++ .../embedded/nativelink-routes.yaml | 17 ++ .../embedded/rebuild-nativelink.yaml | 12 ++ native-cli/components/embedded/trigger.yaml | 2 +- .../embedded/update-image-tags.yaml | 57 ++++++ native-cli/components/flux.go | 31 +++ native-cli/components/loadbalancer.go | 1 + native-cli/default.nix | 2 +- native-cli/go.mod | 16 +- native-cli/go.sum | 32 +-- native-cli/programs/local.go | 24 +++ tools/pre-commit-hooks.nix | 2 +- 61 files changed, 964 insertions(+), 604 deletions(-) rename deployment-examples/chromium/03_build_chrome_tests.sh => deploy/chromium-example/build_chromium_tests.sh (100%) create mode 100644 deploy/chromium-example/kustomization.yaml create mode 100644 deploy/dev/kustomization.yaml create mode 100644 deploy/kubernetes-example/kustomization.yaml delete mode 100644 deployment-examples/chromium/.gitignore delete mode 100755 deployment-examples/chromium/01_operations.sh delete mode 100755 deployment-examples/chromium/02_application.sh delete mode 100755 deployment-examples/chromium/04_delete_application.sh delete mode 100644 deployment-examples/chromium/README.md delete mode 100644 deployment-examples/kubernetes/.gitignore delete mode 100755 deployment-examples/kubernetes/01_operations.sh delete mode 100755 deployment-examples/kubernetes/02_application.sh delete mode 100755 deployment-examples/kubernetes/03_delete_application.sh delete mode 100644 deployment-examples/kubernetes/README.md delete mode 100644 deployment-examples/kubernetes/base/kustomization.yaml delete mode 100644 deployment-examples/kubernetes/worker-lre-java.yaml create mode 100644 docs/src/content/docs/guides/chromium.mdx create mode 100644 docs/src/content/docs/guides/kubernetes.mdx create mode 100644 kubernetes/README.md rename {deployment-examples/kubernetes => kubernetes}/base/cas.yaml (100%) create mode 100644 kubernetes/base/kustomization.yaml rename {deployment-examples/kubernetes => kubernetes}/base/scheduler.yaml (100%) rename deployment-examples/chromium/worker-chromium.yaml => kubernetes/base/worker.yaml (75%) create mode 100644 kubernetes/components/gateway-routes/kustomization.yaml rename {deployment-examples/kubernetes/base => kubernetes/components/gateway-routes}/routes.yaml (100%) rename {deployment-examples/kubernetes/base => kubernetes/components/insecure-certs}/example-do-not-use-in-prod-key.pem (100%) rename {deployment-examples/kubernetes/base => kubernetes/components/insecure-certs}/example-do-not-use-in-prod-rootca.crt (100%) create mode 100644 kubernetes/components/insecure-certs/kustomization.yaml create mode 100644 kubernetes/components/operator/flux-config.yaml create mode 100644 kubernetes/components/operator/kustomization.yaml rename {deployment-examples/kubernetes/base => kubernetes/configmaps}/cas.json (100%) create mode 100644 kubernetes/configmaps/kustomization.yaml rename {deployment-examples/kubernetes/base => kubernetes/configmaps}/scheduler.json (100%) rename {deployment-examples/kubernetes/base => kubernetes/configmaps}/worker.json (100%) create mode 100644 kubernetes/overlays/chromium/kustomization.yaml create mode 100644 kubernetes/overlays/lre/kustomization.yaml rename {deployment-examples/kubernetes => kubernetes/overlays/lre}/worker-lre-cc.yaml (63%) create mode 100644 native-cli/components/embedded/capacitor.yaml create mode 100644 native-cli/components/embedded/update-image-tags.yaml create mode 100644 native-cli/components/flux.go diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 14083e8cc4..351d87878a 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -5,6 +5,7 @@ Cloudflare ELB GPUs Goma +Kustomization LLD LLVM Machina diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index bdcab0adf4..cf5f76d11a 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -78,19 +78,38 @@ jobs: uses: >- # v4 DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41 - - name: Start Kubernetes cluster (Infra) + - name: Start Kubernetes cluster run: > nix run .#native up - - name: Start Kubernetes cluster (Operations) + - name: Start NativeLink operator run: > nix develop --impure --command - bash -c "./deployment-examples/kubernetes/01_operations.sh" + bash -c "kubectl apply -k deployment-examples/kubernetes" - - name: Start Kubernetes cluster (Application) + - name: Wait for Tekton pipelines run: > nix develop --impure --command - bash -c "./deployment-examples/kubernetes/02_application.sh" + bash -c "kubectl wait \ + --for=condition=Succeeded \ + --timeout=45m \ + pipelinerun \ + -l tekton.dev/pipeline=rebuild-nativelink" + + - name: Wait for CAS + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-cas" + + - name: Wait for scheduler + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-scheduler" + + - name: Wait for worker + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-worker" - name: Get gateway IPs id: gateway-ips diff --git a/README.md b/README.md index 7a953842d2..a0040605c4 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Below, you will find a few different options for getting started with NativeLink ## ๐Ÿš€ Example Deployments -You can find a few example deployments in the [deployment-examples directory](./deployment-examples). +You can find a few example deployments in the [Docs](https://docs.nativelink.com/guides/kubernetes). ### ๐Ÿ“ Clone the NativeLink repository 1. Go to the [NativeLink](https://github.com/TraceMachina/nativelink) repository on GitHub. Clone the repository via SSH or HTTPS. In this example the repository is cloned via SSH: diff --git a/deployment-examples/chromium/03_build_chrome_tests.sh b/deploy/chromium-example/build_chromium_tests.sh similarity index 100% rename from deployment-examples/chromium/03_build_chrome_tests.sh rename to deploy/chromium-example/build_chromium_tests.sh diff --git a/deploy/chromium-example/kustomization.yaml b/deploy/chromium-example/kustomization.yaml new file mode 100644 index 0000000000..a6f329d510 --- /dev/null +++ b/deploy/chromium-example/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/chromium + target: + kind: Kustomization + name: nativelink +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/aaronmondal/nativelink + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/ref/branch + value: flux + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml new file mode 100644 index 0000000000..acc552ec93 --- /dev/null +++ b/deploy/dev/kustomization.yaml @@ -0,0 +1,69 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +# Change this value to deploy custom overlays. +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/lre + target: + kind: Kustomization + name: nativelink + +# Modify this value to change the URL of the repository with deployment files. +# +# This is usually only necessary if you change deployment YAML files or +# NativeLink config files. If you only intend to change the Rust sources you can +# leave this as is and need to ensure that the Alerts below are patched to build +# your local sources. +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/TraceMachina/nativelink + target: + kind: GitRepository + name: nativelink + +# Patch this to change to track a custom branch. +- patch: |- + - op: replace + path: /spec/ref/branch + value: main + target: + kind: GitRepository + name: nativelink + +# Setting the flake outputs to `./src_root#xxx` causes the Tekton pipelines to +# build nativelink from your local sources. +# +# During development, the following formats might be useful as well: +# +# `github:user/repo#outname` to build an image from an arbitrary flake output. +# +# `github:TraceMachina/nativelink?ref=pull//head#` to deploy a +# outputs from a Pull request. +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#image + target: + kind: Alert + name: nativelink-image-alert +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#nativelink-worker-init + target: + kind: Alert + name: nativelink-worker-init-alert +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#nativelink-worker-lre-cc + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deploy/kubernetes-example/kustomization.yaml b/deploy/kubernetes-example/kustomization.yaml new file mode 100644 index 0000000000..89fb0b7989 --- /dev/null +++ b/deploy/kubernetes-example/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/lre + target: + kind: Kustomization + name: nativelink +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/aaronmondal/nativelink + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/ref/branch + value: flux + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: github:TraceMachina/nativelink#nativelink-worker-lre-cc + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deployment-examples/chromium/.gitignore b/deployment-examples/chromium/.gitignore deleted file mode 100644 index e0e8ebd5ba..0000000000 --- a/deployment-examples/chromium/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by 02_application.yaml -/kustomization.yaml diff --git a/deployment-examples/chromium/01_operations.sh b/deployment-examples/chromium/01_operations.sh deleted file mode 100755 index cd57c146f4..0000000000 --- a/deployment-examples/chromium/01_operations.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Trigger cluster-internal pipelines to build or fetch necessary images. - -set -xeuo pipefail - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#image"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-init"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-siso-chromium"}' \ - localhost:8082/eventlistener - -until kubectl get pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink | grep -q 'NAME'; do - echo "Waiting for PipelineRuns to start..." - sleep 0.1 -done - -printf "Waiting for PipelineRuns to finish... - -You may cancel this script now and use 'tkn pr ls' and 'tkn pr logs -f' to -monitor the PipelineRun logs. - -" - -kubectl wait \ - --for=condition=Succeeded \ - --timeout=45m \ - pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink diff --git a/deployment-examples/chromium/02_application.sh b/deployment-examples/chromium/02_application.sh deleted file mode 100755 index 6d1e717e67..0000000000 --- a/deployment-examples/chromium/02_application.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Prepare the Kustomization and apply it to the cluster. - -KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/chromium - -cat < "$KUSTOMIZE_DIR"/kustomization.yaml ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: - - ../kubernetes/base - -resources: - - worker-chromium.yaml -EOF - -cd "$KUSTOMIZE_DIR" && kustomize edit set image \ - nativelink=localhost:5001/nativelink:"$(\ - nix eval .#image.imageTag --raw)" \ - nativelink-worker-init=localhost:5001/nativelink-worker-init:"$(\ - nix eval .#nativelink-worker-init.imageTag --raw)" \ - nativelink-worker-chromium=localhost:5001/nativelink-worker-siso-chromium:"$(\ - nix eval .#nativelink-worker-siso-chromium.imageTag --raw)" - -kubectl apply -k "$KUSTOMIZE_DIR" - -kubectl rollout status deploy/nativelink-cas -kubectl rollout status deploy/nativelink-scheduler -kubectl rollout status deploy/nativelink-worker-chromium diff --git a/deployment-examples/chromium/04_delete_application.sh b/deployment-examples/chromium/04_delete_application.sh deleted file mode 100755 index 681370f921..0000000000 --- a/deployment-examples/chromium/04_delete_application.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Delete the Kustomization but leave the rest of the cluster intact. - -kubectl delete -k \ - "$(git rev-parse --show-toplevel)/deployment-examples/chromium" diff --git a/deployment-examples/chromium/README.md b/deployment-examples/chromium/README.md deleted file mode 100644 index bdadb379c6..0000000000 --- a/deployment-examples/chromium/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Chromium example - -This deployment sets up a 4-container deployment with separate CAS, scheduler -and worker. Don't use this example deployment in production. It's insecure. - -> [!WARNING] -> - The client build request is best done from a Ubuntu image, `./03_build_chrome_tests.sh`. It will check if the image is Ubuntu and -> fail otherwise. -> - This tutorial has been tested in a Nix environment of version `2. -> 21.0`. -> - You need to install the [Docker](https://docs.docker.com/engine/install/ubuntu/) Engine in Ubuntu. -> - To get your Nix environment set up see the [official Nix installation documentation](https://nix.dev/install-nix). - -All commands should be run from nix to ensure all dependencies exist in the environment. - -```bash -nix develop -``` - -In this example we're using `kind` to set up the cluster `cilium` to provide a -`LoadBalancer` and `GatewayController`. - -First set up a local development cluster: - -```bash -native up -``` - -> [!TIP] -> The `native up` command uses Pulumi under the hood. You can view and delete -> the stack with `pulumi stack` and `pulumi destroy`. - -Next start a few standard deployments. This part also builds the remote -execution containers and makes them available to the cluster: - -```bash -./01_operations.sh -``` - -> [!TIP] -> The operations invoke cluster-internal Tekton Pipelines to build and push the -> `nativelink` and worker images. You can view the state of the pipelines with -> `tkn pr ls` and `tkn pr logs`/`tkn pr logs --follow`. - -Finally, deploy NativeLink: - -```bash -./02_application.sh -``` - -> [!TIP] -> You can use `./04_delete_application.sh` to remove just the `nativelink` -> deployments but leave the rest of the cluster intact. - -This demo setup creates two gateways to expose the `cas` and `scheduler` -deployments via your local docker network: - -```bash -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') - -echo "Cache IP: $CACHE" -echo "Scheduler IP: $SCHEDULER" -``` - -Using `./03_build_chrome_tests.sh` example script will download needed dependencies -for building Chromium unit tests using NativeLink CAS and Scheduler. The initial part -of the script checks if some dependencies exist, if not installs them, then moves on -to downloading and building Chromium tests. The script simplifies the setup described -in [linux/build_instructions.md](https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md) - -```bash -./03_build_chrome_tests.sh -``` - -> [!TIP] -> You can monitor the logs of container groups with `kubectl logs`: -> ```bash -> kubectl logs -f -l app=nativelink-cas -> kubectl logs -f -l app=nativelink-scheduler -> kubectl logs -f -l app=nativelink-worker-chromium --all-containers=true -> watch $HOME/chromium/src/buildtools/reclient/reproxystatus -> ``` - -When you're done testing, delete the cluster: - -```bash -kind delete cluster -``` -## NativeLink Community -If you have any questions, please reach out to the [NativeLink Community](https://join.slack.com/t/nativelink/shared_invite/zt-2i2mipfr5-lZAEeWYEy4Eru94b3IOcdg). diff --git a/deployment-examples/kubernetes/.gitignore b/deployment-examples/kubernetes/.gitignore deleted file mode 100644 index e0e8ebd5ba..0000000000 --- a/deployment-examples/kubernetes/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by 02_application.yaml -/kustomization.yaml diff --git a/deployment-examples/kubernetes/01_operations.sh b/deployment-examples/kubernetes/01_operations.sh deleted file mode 100755 index 6265ea11d0..0000000000 --- a/deployment-examples/kubernetes/01_operations.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Trigger cluster-internal pipelines to build or fetch necessary images. - -set -xeuo pipefail - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#image"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-init"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-lre-cc"}' \ - localhost:8082/eventlistener - -until kubectl get pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink | grep -q 'NAME'; do - echo "Waiting for PipelineRuns to start..." - sleep 0.1 -done - -printf "Waiting for PipelineRuns to finish... - -You may cancel this script now and use 'tkn pr ls' and 'tkn pr logs -f' to -monitor the PipelineRun logs. - -" - -kubectl wait \ - --for=condition=Succeeded \ - --timeout=45m \ - pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink diff --git a/deployment-examples/kubernetes/02_application.sh b/deployment-examples/kubernetes/02_application.sh deleted file mode 100755 index b444cf9794..0000000000 --- a/deployment-examples/kubernetes/02_application.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Prepare the Kustomization and apply it to the cluster. - -KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/kubernetes - -cat < "$KUSTOMIZE_DIR"/kustomization.yaml ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: - - base - -resources: - - worker-lre-cc.yaml - # TODO(aaronmondal): Fix java and add this: - # - worker-lre-java.yaml -EOF - -cd "$KUSTOMIZE_DIR" && kustomize edit set image \ - nativelink=localhost:5001/nativelink:"$(\ - nix eval .#image.imageTag --raw)" \ - nativelink-worker-init=localhost:5001/nativelink-worker-init:"$(\ - nix eval .#nativelink-worker-init.imageTag --raw)" \ - nativelink-worker-lre-cc=localhost:5001/nativelink-worker-lre-cc:"$(\ - nix eval .#nativelink-worker-lre-cc.imageTag --raw)" - -# TODO(aaronmondal): Fix java and add this: -# nativelink-worker-lre-java=localhost:5001/nativelink-worker-lre-java:$(\ -# nix eval .#nativelink-worker-lre-java.imageTag --raw) - -kubectl apply -k "$KUSTOMIZE_DIR" - -kubectl rollout status deploy/nativelink-cas -kubectl rollout status deploy/nativelink-scheduler -kubectl rollout status deploy/nativelink-worker-lre-cc - -# TODO(aaronmondal): Fix java and add this: -# kubectl rollout status deploy/nativelink-worker-lre-java diff --git a/deployment-examples/kubernetes/03_delete_application.sh b/deployment-examples/kubernetes/03_delete_application.sh deleted file mode 100755 index c26119e863..0000000000 --- a/deployment-examples/kubernetes/03_delete_application.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Delete the Kustomization but leave the rest of the cluster intact. - -kubectl delete -k \ - "$(git rev-parse --show-toplevel)/deployment-examples/kubernetes" diff --git a/deployment-examples/kubernetes/README.md b/deployment-examples/kubernetes/README.md deleted file mode 100644 index 9fdc8510b1..0000000000 --- a/deployment-examples/kubernetes/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# Kubernetes example - -This deployment sets up a 4-container deployment with separate CAS, scheduler -and worker. Don't use this example deployment in production. It's insecure. - -In this example we're using `kind` to set up the cluster `cilium` to provide a -`LoadBalancer` and `GatewayController`. - -First set up a local development cluster: - -```bash -native up -``` - -> [!TIP] -> The `native up` command uses Pulumi under the hood. You can view and delete -> the stack with `pulumi stack` and `pulumi destroy`. - -Next start a few standard deployments. This part also builds the remote -execution containers and makes them available to the cluster: - -```bash -./01_operations.sh -``` - -> [!TIP] -> The operations invoke cluster-internal Tekton Pipelines to build and push the -> `nativelink` and worker images. You can view the state of the pipelines with -> `tkn pr ls` and `tkn pr logs`/`tkn pr logs --follow`. - -Finally, deploy NativeLink: - -```bash -./02_application.sh -``` - -> [!TIP] -> You can use `./03_delete_application.sh` to remove just the `nativelink` -> deployments but leave the rest of the cluster intact. - -This demo setup creates two gateways to expose the `cas` and `scheduler` -deployments via your local docker network: - -```bash -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') - -echo "Cache IP: $CACHE" -echo "Scheduler IP: $SCHEDULER" - -# Prints something like: -# -# Cache IP: 172.20.255.4 -# Scheduler IP: 172.20.255.5 -``` - -You can now pass these IP addresses to your Bazel invocation to use the remote -cache and executor: - -```bash -bazel build \ - --config=lre \ - --remote_instance_name=main \ - --remote_cache=grpc://$CACHE \ - --remote_executor=grpc://$SCHEDULER \ - //local-remote-execution/examples:hello_lre -``` - -> [!TIP] -> You can add these flags to a to a `.bazelrc.user` file in the workspace root. -> Note that you'll need to pass in explicit IP addresses as this file can't -> resolve environment variables: -> ```bash -> # .bazelrc.user -> build --config=lre -> build --remote_instance_name=main -> build --remote_cache=grpc://172.20.255.4 -> build --remote_executor=grpc://172.20.255.5 -> ``` - -When you're done testing, delete the cluster: - -```bash -kind delete cluster -``` - -## Use a published image - -[Published images](https://github.com/TraceMachina/nativelink/pkgs/container/nativelink) can be found under the Container registry, which uses the namespace `https://ghcr.io`. When using the Container registry, you can select prebuilt images and avoid building the image yourself. - -To pull an existing image, you can run: - -```sh -docker pull ghcr.io/tracemachina/nativelink:taggedImageVersion -``` - -## Derive a Tag for an OCI image - -To derive the tag of the NativeLink image at a specific commit, run the below command and change `someCommit` with the commit hash you want to use: - -```sh -nix eval github:TraceMachina/nativelink/someCommit#image.imageTag --raw -``` - -Alternatively, the tag can be derived from the upstream sources at the current state of the upstream main branch by running this command: - -```sh -nix eval github:TraceMachina/nativelink#image.imageTag --raw -``` - -Similarly, you can also clone or checkout a specific version or commit of the NativeLink git repository to evaluate the output of the entire NativeLink flake. For example, assuming you've done the [NativeLink Getting Started Guide](https://github.com/TraceMachina/nativelink?tab=readme-ov-file#getting-started-with-nativelink) and cloned the repository, you can run these sample commands: - -```sh -git log -git checkout commitHash -nix eval .#image.imageTag --raw -``` -The `--raw` removes the surrounding quotes from the output string. - -> [!WARNING] -> We don't recommend using this command to -> retrieve an image: -> ```sh -> nix eval github:TraceMachina/nativelink#image.imageTag --raw -> ``` -> Using this command prevents anyone from -> identifying the specific version of the -> NativeLink container in use because -> reflects the image version available at the -> time of download. It'll be hard to debug, -> revert to previous versions if there are issues -> and complicate bug tracking. -> It's for these same reasons you won't be able -> to retrieve an image using the `latest` tag. - -## Build and copy an OCI image - -You can build and copy the image to a container registry with the `copyTo` attribute. Below are examples within the NativeLink repository for building and copying an image: -- [Example 1](https://github.com/TraceMachina/nativelink/blob/09b32c94d3cc7780816585e9b87f69c56cf931ae/deployment-examples/kubernetes/01_operations.sh#L12-L16) highlights: - -```sh -nix run github:tracemachina/nativelink#image.copyTo -``` - -- [Example 2](https://github.com/TraceMachina/nativelink/blob/09b32c94d3cc7780816585e9b87f69c56cf931ae/tools/local-image-test.nix#L12-L13) highlights how to skip pushing to an intermediary registry by copying directly to the docker-daemon: - -```sh -IMAGE_NAME=$(nix eval .#image.imageName --raw) -IMAGE_TAG=$(nix eval .#image.imageTag --raw) - -nix run .#image.copyTo docker-daemon:"${IMAGE_NAME}":"${IMAGE_TAG}" -``` - -You can find more about details around [nix](https://github.com/nlewo/nix2container). Published images are signed using `cosign`. For more details of the verification process of publishing OCI images see [SECURITY.md](https://github.com/TraceMachina/nativelink/blob/main/SECURITY.md) - -## NativeLink Community - -Reach out to the [NativeLink Slack community](https://join.slack.com/t/nativelink/shared_invite/zt-2forhp5n9-L7dTD21nCSY9_IRteQvZmw) for any questions via #NativeLink! diff --git a/deployment-examples/kubernetes/base/kustomization.yaml b/deployment-examples/kubernetes/base/kustomization.yaml deleted file mode 100644 index 8bc3a9d247..0000000000 --- a/deployment-examples/kubernetes/base/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - cas.yaml - - scheduler.yaml - - routes.yaml - -configMapGenerator: - - name: cas - files: - - cas.json - - name: scheduler - files: - - scheduler.json - - name: worker - files: - - worker.json - -images: - - name: nativelink - -secretGenerator: - - name: tls-secret - files: - - example-do-not-use-in-prod-rootca.crt - - example-do-not-use-in-prod-key.pem diff --git a/deployment-examples/kubernetes/worker-lre-java.yaml b/deployment-examples/kubernetes/worker-lre-java.yaml deleted file mode 100644 index d4c5804810..0000000000 --- a/deployment-examples/kubernetes/worker-lre-java.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nativelink-worker-lre-java -spec: - replicas: 1 - selector: - matchLabels: - app: nativelink-worker-lre-java - template: - metadata: - labels: - app: nativelink-worker-lre-java - spec: - initContainers: - - name: setup-entrypoint - image: nixpkgs/nix-flakes:latest - command: ["/bin/sh", "-c"] - # The kind setup mounts the nativelink repository into the kind nodes at - # `/mnt/src_root`. This ensures that the tags between the worker configs - # and bazel toolchains match when this setup is run in CI. - # - # WARNING: The platform is *not* necessarily the container that is - # actually deployed here. The generator container in this example was - # `rbe-autogen-lre-java:` and the platform was modified - # after the fact to be `lre-java:`. The deployed container - # we use as worker is - # `nativelink-worker-lre-java:` which is a - # completely separate extension of the `lre-java` base image. - args: - - | - git config --global --add safe.directory "*" - NATIVELINK_WORKER_PLATFORM=docker://lre-java:$(nix eval /mnt/src_root#lre-java.imageTag --raw) && - printf '#!/bin/sh\nexport NATIVELINK_WORKER_PLATFORM=%s\nexec "$@"' "$NATIVELINK_WORKER_PLATFORM" > /entrypoint/entrypoint.sh && - chmod +x /entrypoint/entrypoint.sh - volumeMounts: - - name: entrypoint - mountPath: /entrypoint - - name: mnt - mountPath: /mnt - containers: - - name: nativelink-worker-lre-java - # This image will be edited by kustomize - image: nativelink-worker-lre-java - env: - - name: RUST_LOG - value: warn - - name: CAS_ENDPOINT - value: nativelink-cas - - name: SCHEDULER_ENDPOINT - value: nativelink-scheduler - volumeMounts: - - name: worker-config - mountPath: /worker.json - subPath: worker.json - - name: entrypoint - mountPath: /entrypoint - command: ["/entrypoint/entrypoint.sh"] - args: ["/bin/nativelink", "/worker.json"] - volumes: - - name: entrypoint - emptyDir: {} - - name: worker-config - configMap: - name: worker - - name: mnt - hostPath: - path: /mnt diff --git a/docs/.gitignore b/docs/.gitignore index 1c24988f18..abc47868d0 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -32,7 +32,5 @@ src/content/docs/explanations/lre.mdx src/content/docs/reference/changelog.mdx src/content/docs/reference/nativelink-config.mdx src/content/docs/tutorials/setup.mdx -src/content/docs/guides/chromium.mdx src/content/docs/guides/configuration.mdx -src/content/docs/guides/kubernetes.mdx src/content/docs/guides/setup.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index faf7cc850e..4c4760bf7a 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -106,16 +106,16 @@ export default defineConfig({ label: "Guides", items: [ { - label: "Configuration examples", - link: "/guides/configuration", + label: "Kubernetes example", + link: "/guides/kubernetes", }, { label: "Chromium example", link: "/guides/chromium", }, { - label: "Kubernetes example", - link: "/guides/kubernetes", + label: "Configuration examples", + link: "/guides/configuration", }, { label: "Contributing", diff --git a/docs/scripts/md_to_mdx_aot.ts b/docs/scripts/md_to_mdx_aot.ts index 613091d81e..e5f23dbce9 100644 --- a/docs/scripts/md_to_mdx_aot.ts +++ b/docs/scripts/md_to_mdx_aot.ts @@ -58,16 +58,6 @@ convertMarkdownToMdx( "src/content/docs/guides/configuration.mdx", "NativeLink configuration guide", ); -convertMarkdownToMdx( - "../deployment-examples/chromium/README.md", - "src/content/docs/guides/chromium.mdx", - "NativeLink deployment example for Chromium", -); -convertMarkdownToMdx( - "../deployment-examples/kubernetes/README.md", - "src/content/docs/guides/kubernetes.mdx", - "NativeLink deployment example for Kubernetes", -); convertMarkdownToMdx( "../CHANGELOG.md", "src/content/docs/reference/changelog.mdx", diff --git a/docs/src/content/docs/guides/chromium.mdx b/docs/src/content/docs/guides/chromium.mdx new file mode 100644 index 0000000000..a1cfd44969 --- /dev/null +++ b/docs/src/content/docs/guides/chromium.mdx @@ -0,0 +1,120 @@ +--- +title: Chromium example +description: 'An example for building Chromium with NativeLink in Kubernetes.' +--- + +In this example you'll spin up a local Kubernetes cluster with NativeLink and +run a Chromium build against it. + +**Requirements** + +- An `x86_64-linux` system running a recent Ubuntu. Either "real" Linux or WSL2. +- A functional local Docker setup. +- A recent version of Nix with flake support, for instance installed via the + [next-gen Nix installer](https://github.com/NixOS/experimental-nix-installer). + +:::caution +This example doesn't work on Mac and Linux distributions other than Ubuntu. +::: + +## โ˜๏ธ Prepare the cluster + +First, enter the NativeLink development environment: + +```bash +# TODO(aaronmondal): Point to the main repo before merging. +git clone https://github.com/aaronmondal/nativelink && \ + cd nativelink && \ + git switch flux \ + nix develop +``` + +This environment contains some cloud tooling, so you don't need to set up any +kubernetes-related software yourself. + +Now, start the development cluster: + +```bash +native up +``` + +:::tip +The `native up` command uses Pulumi under the hood. You can view and delete +the stack with `pulumi stack` and `pulumi destroy`. If you're queried for a +stack password, press enter, as the password is an empty string. +::: + +Next, deploy NativeLink to the cluster: + +```bash +kubectl apply -k \ + https://github.com/aaronmondal/nativelink//deploy/chromium-example?ref=flux +``` + +:::danger +This example is built for demo purposes only. It's not a secure production-grade +setup and will only work in the local development cluster created with +`native up`. + +One-liner production-grade setups are still under construction. +::: + +## ๐Ÿ”ญ Explore deployments + +The deployment might take a wile to boot up. You can monitor progress via the +dashboards that come with the development cluster: + +- [localhost:8080](http://localhost:8080): Cilium's Hubble UI to view the + cluster topology. NativeLink will be deployed into the `default` namespace. +- [localhost:8081](http://localhost:8081): The Tekton Dashboard to view the + progress of the in-cluster pipelines. You'll find the pipelines under the + `PipelineRuns` tab. +- [localhost:9000](http://localhost:9000): The Capacitor Dashboard to view Flux + Kustomizations. You can view NatieLink's logs here once it's fully deployed. + +In terminals, the following commands can be helpful to view deployment progress: + +- `tkn pr logs -f` to view the logs of a `PipelineRun` in the terminal. +- `flux get all -A` to view the state of the NativeLink deployments. +- Once NativeLink is deployed: + - `kubectl logs deploy/nativelink-cas` for the CAS (cache) logs. + - `kubectl logs deploy/nativelink-scheduler` for the scheduler logs. + - `kubectl logs deploy/nativelink-worker` for the worker logs. + +## ๐Ÿ—๏ธ Build against NativeLink + +The demo setup creates gateways to expose the `cas` and `scheduler` deployments +via your local docker network. The following command builds the Chromium tests +against the cluster: + +```bash +build-chromium-tests +``` + +The `build-chromium-tests` command simplifies the setup described in +[linux/build_instructions.md](https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md). +After preparing the requirements, it runs a Reclient build against the cluster. + +:::note +See [`deploy/chromium-example/build_chromium_tests.sh`](https://github.com/TraceMachina/nativelink/blob/main/deploy/chromium-example/build_chromium_tests.sh) +for the script contents. +::: + +You can view Reclient's logs like so: + +```bash +watch $HOME/chromium/src/buildtools/reclient/reproxystatus +``` + +## ๐Ÿงน Clean up + +When you're done testing, delete the cluster: + +```bash +# Delete the kind cluster +native down + +# Remove the container registry and loadbalancer +docker container stop kind-registry | xargs docker rm +docker container stop kind-loadbalancer | xargs docker rm +``` diff --git a/docs/src/content/docs/guides/kubernetes.mdx b/docs/src/content/docs/guides/kubernetes.mdx new file mode 100644 index 0000000000..d1ab3b81f5 --- /dev/null +++ b/docs/src/content/docs/guides/kubernetes.mdx @@ -0,0 +1,191 @@ +--- +title: Kubernetes example +description: 'An example setup for NativeLink in Kubernetes' +--- + +In this example you'll spin up a local Kubernetes cluster with NativeLink and +run some Bazel builds against it. + +**Requirements** + +- An `x86_64-linux` system. Either "real" Linux or WSL2. +- A functional local Docker setup. +- A recent version of Nix with flake support, for instance installed via the + [next-gen Nix installer](https://github.com/NixOS/experimental-nix-installer). + +:::caution +This example doesn't work on Mac yet. +::: + + +## โ˜๏ธ Prepare the cluster + +First, enter the NativeLink development environment: + +```bash +# TODO(aaronmondal): Point to the main repo before merging. +git clone https://github.com/aaronmondal/nativelink && \ + cd nativelink && \ + git switch flux \ + nix develop +``` + +This environment contains Bazel and some cloud tooling, so you don't need to set +up any kubernetes-related software yourself. + +Now, start the development cluster: + +```bash +native up +``` + +:::tip +The `native up` command uses Pulumi under the hood. You can view and delete +the stack with `pulumi stack` and `pulumi destroy`. If you're queried for a +stack password, press enter, as the password is an empty string. +::: + +Next, deploy NativeLink to the cluster: + +```bash +kubectl apply -k \ + https://github.com/aaronmondal/nativelink//deploy/kubernetes-example?ref=flux +``` + +:::danger +This example is built for demo purposes only. It's not a secure production-grade +setup and will only work in the local development cluster created with +`native up`. + +One-liner production-grade setups are still under construction. +::: + +## ๐Ÿ”ญ Explore deployments + +The deployment might take a wile to boot up. You can monitor progress via the +dashboards that come with the development cluster: + +- [localhost:8080](http://localhost:8080): Cilium's Hubble UI to view the + cluster topology. NativeLink will be deployed into the `default` namespace. +- [localhost:8081](http://localhost:8081): The Tekton Dashboard to view the + progress of the in-cluster pipelines. You'll find the pipelines under the + `PipelineRuns` tab. +- [localhost:9000](http://localhost:9000): The Capacitor Dashboard to view Flux + Kustomizations. You can view NatieLink's logs here once it's fully deployed. + +In terminals, the following commands can be helpful to view deployment progress: + +- `tkn pr logs -f` to view the logs of a `PipelineRun` in the terminal. +- `flux get all -A` to view the state of the NativeLink deployments. +- Once NativeLink is deployed: + - `kubectl logs deploy/nativelink-cas` for the CAS (cache) logs. + - `kubectl logs deploy/nativelink-scheduler` for the scheduler logs. + - `kubectl logs deploy/nativelink-worker` for the worker logs. + +## ๐Ÿ—๏ธ Build against NativeLink + +The demo setup creates gateways to expose the `cas` and `scheduler` deployments +via your local docker network. You can pass the Gateway addresses to Bazel +invocations to make builds run against the cluster: + +```bash +CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') +SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') + +echo "Cache IP: $CACHE" +echo "Scheduler IP: $SCHEDULER" + +bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + --remote_executor=grpc://$SCHEDULER \ + //local-remote-execution/examples:hello_lre +``` + +:::caution +While the Dashboard ports are static, the NativeLink endpoints aren't (yet). +If you shut down the cluster and reboot it, the `$CACHE` and `$SCHEDULER` IP +addresses will change. +::: + +:::tip +You can add these flags to a to a `.bazelrc.user` file in the workspace root. +Note that you'll need to pass in explicit IP addresses as this file can't +resolve environment variables: +```bash +# .bazelrc.user +build --config=lre +build --remote_instance_name=main +build --remote_cache=grpc://172.20.255.4 +build --remote_executor=grpc://172.20.255.5 +``` + +```bash +# .bazelrc +try-import %workspace%/.bazelrc.user +``` +::: + +The crucial part is this bit: + +```txt +INFO: 11 processes: 9 internal, 2 remote. +``` + +It tells us that the compilation ran against the cluster. Let's clean the Bazel +cache and run the build again: + +```bash +bazel clean && bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + --remote_executor=grpc://$SCHEDULER \ + //local-remote-execution/examples:hello_lre +``` + +The build now shows cache hits instead of remote actions: + +```txt +INFO: 11 processes: 2 remote cache hit, 9 internal. +``` + +## ๐Ÿš€ Bonus: Local Remote Execution + +The worker deployment in this example leverages [Local Remote Execution](../explanations/lre). + +Local Remote Execution mirrors toolchains for remote execution in your local +development environment. This lets you reuse build artifacts with virtually +perfect cache hit rate across different repositories, developers, and CI. + +To test LRE in the cluster, clean the local cache and invoke another build +against the cluster, but this time omit the `remote_executor` flag. This way +you'll use remote caching without remote execution: + +```bash +bazel clean && bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + //local-remote-execution/examples:hello_lre +``` + +You'll get remote cache hits as if your local machine was a `nativelink-worker`: + +```txt +INFO: 11 processes: 2 remote cache hit, 9 internal. +``` + +## ๐Ÿงน Clean up + +When you're done testing, delete the cluster: + +```bash +# Delete the kind cluster +native down + +# Remove the container registry and loadbalancer +docker container stop kind-registry | xargs docker rm +docker container stop kind-loadbalancer | xargs docker rm +``` diff --git a/flake.nix b/flake.nix index 8597e051d1..d69c422c83 100644 --- a/flake.nix +++ b/flake.nix @@ -145,6 +145,11 @@ native-cli = import ./native-cli/default.nix {inherit pkgs;}; + build-chromium-tests = + pkgs.writeShellScriptBin + "build-chromium-tests" + ./deploy/chromium-example/build_chromium_tests.sh; + docs = pkgs.callPackage ./tools/docs.nix {rust = stable-rust.default;}; inherit (nix2container.packages.${system}.nix2container) pullImage; @@ -283,6 +288,7 @@ pkgs.docker-client pkgs.kind pkgs.tektoncd-cli + pkgs.fluxcd (pkgs.pulumi.withPackages (ps: [ps.pulumi-language-go])) pkgs.go pkgs.kustomize @@ -294,6 +300,7 @@ customClang native-cli docs + build-chromium-tests ] ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [ # The docs on Mac require a manual setup outside the flake. diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000000..dd516969af --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,13 @@ +# NativeLink Kubernetes deployments + +Building blocks for NativeLink Kubernetes deployments. + +This directory does **not** contain a one-size-fits-all solution like a Helm +chart - infrastructure requirements are too diverse for a single setup to +reliably cover all potential use-cases. + +Instead, we provide useful building blocks in the form of Kustomizations. +Downstream implementers might use them as reference points to patch in the +functionality they require. + +See the `deployment-examples` directory for concrete example deployments. diff --git a/deployment-examples/kubernetes/base/cas.yaml b/kubernetes/base/cas.yaml similarity index 100% rename from deployment-examples/kubernetes/base/cas.yaml rename to kubernetes/base/cas.yaml diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000000..92d4e29f7d --- /dev/null +++ b/kubernetes/base/kustomization.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cas.yaml + - scheduler.yaml + - worker.yaml + - ../configmaps + +images: + - name: nativelink + - name: nativelink-worker-init + - name: nativelink-worker diff --git a/deployment-examples/kubernetes/base/scheduler.yaml b/kubernetes/base/scheduler.yaml similarity index 100% rename from deployment-examples/kubernetes/base/scheduler.yaml rename to kubernetes/base/scheduler.yaml diff --git a/deployment-examples/chromium/worker-chromium.yaml b/kubernetes/base/worker.yaml similarity index 75% rename from deployment-examples/chromium/worker-chromium.yaml rename to kubernetes/base/worker.yaml index 29ca10a7ff..dcf57bc2c4 100644 --- a/deployment-examples/chromium/worker-chromium.yaml +++ b/kubernetes/base/worker.yaml @@ -2,16 +2,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker-chromium + name: nativelink-worker spec: replicas: 3 selector: matchLabels: - app: nativelink-worker-chromium + app: nativelink-worker template: metadata: labels: - app: nativelink-worker-chromium + app: nativelink-worker spec: initContainers: - name: nativelink-worker-init @@ -21,11 +21,10 @@ spec: volumeMounts: - name: shared mountPath: /shared - containers: - - name: nativelink-worker-chromium + - name: nativelink-worker # This image will be edited by kustomize. - image: nativelink-worker-chromium + image: nativelink-worker env: - name: RUST_LOG value: info @@ -37,6 +36,8 @@ spec: - name: worker-config mountPath: /worker.json subPath: worker.json + - name: shared + mountPath: /shared command: ["/shared/nativelink"] args: ["/worker.json"] volumes: @@ -45,3 +46,12 @@ spec: - name: worker-config configMap: name: worker +--- +apiVersion: v1 +kind: Service +metadata: + name: nativelink-worker +spec: + selector: + app: nativelink-worker + clusterIP: None diff --git a/kubernetes/components/gateway-routes/kustomization.yaml b/kubernetes/components/gateway-routes/kustomization.yaml new file mode 100644 index 0000000000..42b112b7e6 --- /dev/null +++ b/kubernetes/components/gateway-routes/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - routes.yaml diff --git a/deployment-examples/kubernetes/base/routes.yaml b/kubernetes/components/gateway-routes/routes.yaml similarity index 100% rename from deployment-examples/kubernetes/base/routes.yaml rename to kubernetes/components/gateway-routes/routes.yaml diff --git a/deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem b/kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem similarity index 100% rename from deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem rename to kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem diff --git a/deployment-examples/kubernetes/base/example-do-not-use-in-prod-rootca.crt b/kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt similarity index 100% rename from deployment-examples/kubernetes/base/example-do-not-use-in-prod-rootca.crt rename to kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt diff --git a/kubernetes/components/insecure-certs/kustomization.yaml b/kubernetes/components/insecure-certs/kustomization.yaml new file mode 100644 index 0000000000..bc92cb026f --- /dev/null +++ b/kubernetes/components/insecure-certs/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +secretGenerator: + - name: tls-secret + files: + - example-do-not-use-in-prod-rootca.crt + - example-do-not-use-in-prod-key.pem diff --git a/kubernetes/components/operator/flux-config.yaml b/kubernetes/components/operator/flux-config.yaml new file mode 100644 index 0000000000..c016867f52 --- /dev/null +++ b/kubernetes/components/operator/flux-config.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: nativelink-webhook + namespace: flux-system +spec: + type: generic + address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-image-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: github:TraceMachina/nativelink#image +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-worker-init-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: github:TraceMachina/nativelink#nativelink-worker-init +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-worker-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: "PLACEHOLDER_NATIVELINK_WORKER" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-configmaps + namespace: default +spec: + interval: 2m + path: "./kubernetes/configmaps" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + path: "PLACEHOLDER_NATIVELINK_KUSTOMIZATION" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substituteFrom: + - kind: ConfigMap + name: nativelink-image-tags + dependsOn: + - name: nativelink-configmaps diff --git a/kubernetes/components/operator/kustomization.yaml b/kubernetes/components/operator/kustomization.yaml new file mode 100644 index 0000000000..916d215e1e --- /dev/null +++ b/kubernetes/components/operator/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: +- flux-config.yaml diff --git a/deployment-examples/kubernetes/base/cas.json b/kubernetes/configmaps/cas.json similarity index 100% rename from deployment-examples/kubernetes/base/cas.json rename to kubernetes/configmaps/cas.json diff --git a/kubernetes/configmaps/kustomization.yaml b/kubernetes/configmaps/kustomization.yaml new file mode 100644 index 0000000000..91752af244 --- /dev/null +++ b/kubernetes/configmaps/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: + - name: cas + files: + - cas.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + - name: scheduler + files: + - scheduler.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + - name: worker + files: + - worker.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled diff --git a/deployment-examples/kubernetes/base/scheduler.json b/kubernetes/configmaps/scheduler.json similarity index 100% rename from deployment-examples/kubernetes/base/scheduler.json rename to kubernetes/configmaps/scheduler.json diff --git a/deployment-examples/kubernetes/base/worker.json b/kubernetes/configmaps/worker.json similarity index 100% rename from deployment-examples/kubernetes/base/worker.json rename to kubernetes/configmaps/worker.json diff --git a/kubernetes/overlays/chromium/kustomization.yaml b/kubernetes/overlays/chromium/kustomization.yaml new file mode 100644 index 0000000000..10837536be --- /dev/null +++ b/kubernetes/overlays/chromium/kustomization.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base + +components: +- ../../components/gateway-routes +- ../../components/insecure-certs + +images: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker + newName: localhost:5001/nativelink-worker-siso-chromium + newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} diff --git a/kubernetes/overlays/lre/kustomization.yaml b/kubernetes/overlays/lre/kustomization.yaml new file mode 100644 index 0000000000..707804b4ef --- /dev/null +++ b/kubernetes/overlays/lre/kustomization.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base + +components: +- ../../components/gateway-routes +- ../../components/insecure-certs + +patches: +- path: worker-lre-cc.yaml + target: + kind: Deployment + name: nativelink-worker + +images: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker + newName: localhost:5001/nativelink-worker-lre-cc + newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} diff --git a/deployment-examples/kubernetes/worker-lre-cc.yaml b/kubernetes/overlays/lre/worker-lre-cc.yaml similarity index 63% rename from deployment-examples/kubernetes/worker-lre-cc.yaml rename to kubernetes/overlays/lre/worker-lre-cc.yaml index 36b79aed64..cbdb14c060 100644 --- a/deployment-examples/kubernetes/worker-lre-cc.yaml +++ b/kubernetes/overlays/lre/worker-lre-cc.yaml @@ -2,16 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker-lre-cc + name: nativelink-worker spec: replicas: 1 - selector: - matchLabels: - app: nativelink-worker-lre-cc template: - metadata: - labels: - app: nativelink-worker-lre-cc spec: initContainers: - name: setup-entrypoint @@ -39,44 +33,16 @@ spec: mountPath: /entrypoint - name: mnt mountPath: /mnt - - - name: nativelink-worker-init - # This image will be edited by kustomize. - image: nativelink-worker-init - args: ["/shared/nativelink"] - volumeMounts: - - name: shared - mountPath: /shared - containers: - - name: nativelink-worker-lre-cc - # This image will be edited by kustomize. - image: nativelink-worker-lre-cc - env: - - name: RUST_LOG - value: warn - - name: CAS_ENDPOINT - value: nativelink-cas - - name: SCHEDULER_ENDPOINT - value: nativelink-scheduler + - name: nativelink-worker volumeMounts: - - name: worker-config - mountPath: /worker.json - subPath: worker.json - name: entrypoint mountPath: /entrypoint - - name: shared - mountPath: /shared command: ["/entrypoint/entrypoint.sh"] args: ["/shared/nativelink", "/worker.json"] volumes: - - name: shared - emptyDir: {} - name: entrypoint emptyDir: {} - - name: worker-config - configMap: - name: worker - name: mnt hostPath: path: /mnt diff --git a/native-cli/clusters/localcluster.go b/native-cli/clusters/localcluster.go index dc7b253237..8b63ad47f9 100644 --- a/native-cli/clusters/localcluster.go +++ b/native-cli/clusters/localcluster.go @@ -10,7 +10,7 @@ import ( "runtime" "text/template" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" git "github.com/go-git/go-git/v5" "sigs.k8s.io/kind/pkg/cluster" @@ -210,7 +210,7 @@ func createRegistryConfigInNode( ) error { config := fmt.Sprintf("[host.\"http://%s:%d\"]", regName, internalPort) regDir := fmt.Sprintf("/etc/containerd/certs.d/localhost:%d", externalPort) - execConfig := types.ExecConfig{ + execConfig := container.ExecOptions{ Cmd: []string{ "sh", "-c", @@ -232,7 +232,7 @@ func createRegistryConfigInNode( ) } - if err := cli.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{}); err != nil { + if err := cli.ContainerExecStart(ctx, execID.ID, container.ExecAttachOptions{}); err != nil { return fmt.Errorf( "error starting exec command on node %s: %w", nodeName, diff --git a/native-cli/components/cilium.go b/native-cli/components/cilium.go index 31e61e1aaa..feb3b7bd79 100644 --- a/native-cli/components/cilium.go +++ b/native-cli/components/cilium.go @@ -8,7 +8,7 @@ import ( "slices" "strings" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" helmv3 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" @@ -134,7 +134,7 @@ func kindIPv4Subnet() (string, error) { return "", fmt.Errorf("%w: %w", errPulumi, err) } - networks, err := cli.NetworkList(dockerCtx, types.NetworkListOptions{}) + networks, err := cli.NetworkList(dockerCtx, network.ListOptions{}) if err != nil { return "", fmt.Errorf("%w: %w", errPulumi, err) } diff --git a/native-cli/components/embedded/capacitor.yaml b/native-cli/components/embedded/capacitor.yaml new file mode 100644 index 0000000000..e9de009148 --- /dev/null +++ b/native-cli/components/embedded/capacitor.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: capacitor + namespace: flux-system +spec: + interval: 12h + url: oci://ghcr.io/gimlet-io/capacitor-manifests + ref: + semver: ">=0.1.0" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: capacitor + namespace: flux-system +spec: + targetNamespace: flux-system + interval: 1h + retryInterval: 2m + timeout: 5m + wait: true + prune: true + path: "./" + sourceRef: + kind: OCIRepository + name: capacitor +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-ingress-to-capacitor + namespace: flux-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: onechart + app.kubernetes.io/instance: capacitor + ingress: + - fromEntities: + - ingress + - toPorts: + - ports: + - port: "9000" + protocol: TCP diff --git a/native-cli/components/embedded/envoy.template.yaml b/native-cli/components/embedded/envoy.template.yaml index 4dbe7051d3..4438265550 100644 --- a/native-cli/components/embedded/envoy.template.yaml +++ b/native-cli/components/embedded/envoy.template.yaml @@ -32,6 +32,8 @@ static_resources: {{- end }} http_filters: - name: envoy.filters.http.router + upgrade_configs: + - upgrade_type: websocket {{- end }} clusters: {{- range .InternalGateways }} diff --git a/native-cli/components/embedded/kustomization.yaml b/native-cli/components/embedded/kustomization.yaml index 945ec1d9ee..ee96870803 100644 --- a/native-cli/components/embedded/kustomization.yaml +++ b/native-cli/components/embedded/kustomization.yaml @@ -7,5 +7,7 @@ resources: - skopeo-check-hashlocked-url.yaml - nix2container-image-info.yaml - trigger.yaml + - update-image-tags.yaml + - capacitor.yaml # - nativelink-gateways.yaml # Gateways are handled in Pulumi via the # NativeLinkGateways resource. diff --git a/native-cli/components/embedded/nativelink-gateways.yaml b/native-cli/components/embedded/nativelink-gateways.yaml index a9ba34bf16..960289ad0a 100644 --- a/native-cli/components/embedded/nativelink-gateways.yaml +++ b/native-cli/components/embedded/nativelink-gateways.yaml @@ -57,3 +57,15 @@ spec: - name: tkn-gateway protocol: HTTP port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: capacitor-gateway + namespace: flux-system +spec: + gatewayClassName: cilium + listeners: + - name: capacitor-gateway + protocol: HTTP + port: 80 diff --git a/native-cli/components/embedded/nativelink-routes.yaml b/native-cli/components/embedded/nativelink-routes.yaml index 0dbc8e8e08..192b098edc 100644 --- a/native-cli/components/embedded/nativelink-routes.yaml +++ b/native-cli/components/embedded/nativelink-routes.yaml @@ -49,3 +49,20 @@ spec: backendRefs: - name: tekton-dashboard port: 9097 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: capacitor-route + namespace: flux-system +spec: + parentRefs: + - sectionName: capacitor-gateway + name: capacitor-gateway + rules: + - matches: + - path: + value: / + backendRefs: + - name: capacitor + port: 9000 diff --git a/native-cli/components/embedded/rebuild-nativelink.yaml b/native-cli/components/embedded/rebuild-nativelink.yaml index 4259377d77..eb540dd78a 100644 --- a/native-cli/components/embedded/rebuild-nativelink.yaml +++ b/native-cli/components/embedded/rebuild-nativelink.yaml @@ -163,3 +163,15 @@ spec: - input: "$(tasks.check-hashlocked-url.results.exists)" operator: notin values: ["true"] + + - name: update-image-tags + taskRef: + name: update-image-tags + params: + - name: imageName + value: "$(tasks.get-image-info.results.imageName)" + - name: imageTag + value: "$(tasks.get-image-info.results.imageTag)" + runAfter: + - copy-verified-prebuilt-image + - copy-nix-built-image diff --git a/native-cli/components/embedded/trigger.yaml b/native-cli/components/embedded/trigger.yaml index 1aaea3b571..229d2648db 100644 --- a/native-cli/components/embedded/trigger.yaml +++ b/native-cli/components/embedded/trigger.yaml @@ -57,7 +57,7 @@ metadata: spec: params: - name: flakeOutput - value: "$(body.flakeOutput)" + value: "$(body.metadata.flakeOutput)" --- apiVersion: v1 kind: ServiceAccount diff --git a/native-cli/components/embedded/update-image-tags.yaml b/native-cli/components/embedded/update-image-tags.yaml new file mode 100644 index 0000000000..6948004571 --- /dev/null +++ b/native-cli/components/embedded/update-image-tags.yaml @@ -0,0 +1,57 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: configmap-manager + namespace: default +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: configmap-manager-binding + namespace: default +subjects: +- kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: Role + name: configmap-manager + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: update-image-tags + labels: + app.kubernetes.io/versions: "0.1" +spec: + description: > + Update or add image tags in a ConfigMap. + params: + - name: imageName + - name: imageTag + steps: + - name: update-configmap + image: bitnami/kubectl + script: | + #!/bin/bash + set -e + + CM_NAME="nativelink-image-tags" + + # Transform the key to a format that flux accepts. The transformation + # here looks like `nativelink-worker` -> `NATIVELINK_WORKER_TAG`. + TRANSFORMED_KEY=$(echo "$(params.imageName)" | tr '[:lower:]' '[:upper:]' | tr '-' '_')_TAG + + # Check if the ConfigMap exists + if kubectl get configmap $CM_NAME &>/dev/null; then + # ConfigMap exists, update it + kubectl patch configmap $CM_NAME --type=json -p='[{"op": "add", "path": "/data/'$TRANSFORMED_KEY'", "value": "$(params.imageTag)"}]' + else + # ConfigMap doesn't exist, create it + kubectl create configmap $CM_NAME --from-literal=$TRANSFORMED_KEY=$(params.imageTag) + fi diff --git a/native-cli/components/flux.go b/native-cli/components/flux.go new file mode 100644 index 0000000000..0677c315a1 --- /dev/null +++ b/native-cli/components/flux.go @@ -0,0 +1,31 @@ +package components + +import ( + "fmt" + + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +// The configuration for Flux. +type Flux struct { + Version string +} + +// Install installs the Flux CRDs into the cluster.. +func (component *Flux) Install( + ctx *pulumi.Context, + name string, +) ([]pulumi.Resource, error) { + flux, err := yaml.NewConfigFile(ctx, name, &yaml.ConfigFileArgs{ + File: fmt.Sprintf( + "https://github.com/fluxcd/flux2/releases/download/v%s/install.yaml", + component.Version, + ), + }) + if err != nil { + return nil, fmt.Errorf("%w: %w", errPulumi, err) + } + + return []pulumi.Resource{flux}, nil +} diff --git a/native-cli/components/loadbalancer.go b/native-cli/components/loadbalancer.go index 7118b89a4b..61f3a421d4 100644 --- a/native-cli/components/loadbalancer.go +++ b/native-cli/components/loadbalancer.go @@ -310,6 +310,7 @@ func (component *Loadbalancer) Install( "el-gateway": false, "hubble-gateway": false, "tkn-gateway": false, + "capacitor-gateway": false, }, ), component.Gateways) if err != nil { diff --git a/native-cli/default.nix b/native-cli/default.nix index ade32b0650..c94c3db11f 100644 --- a/native-cli/default.nix +++ b/native-cli/default.nix @@ -3,7 +3,7 @@ pkgs.buildGoModule { pname = "native-cli"; version = "0.4.0"; src = ./.; - vendorHash = "sha256-zB+gaJB+5KEnkPHX2BY8nbO/oOmPk4lfmGzdPBMOSxE="; + vendorHash = "sha256-eVfeZiyyO7rUH1kDeEa41uWA0ERwCPFnOu6VTsBIEUg="; buildInputs = [pkgs.makeWrapper]; ldflags = ["-s -w"]; installPhase = '' diff --git a/native-cli/go.mod b/native-cli/go.mod index d3fa6d457c..9f048b9073 100644 --- a/native-cli/go.mod +++ b/native-cli/go.mod @@ -3,11 +3,11 @@ module github.com/TraceMachina/nativelink/native-cli go 1.22.1 require ( - github.com/docker/docker v27.0.0+incompatible + github.com/docker/docker v27.0.2+incompatible github.com/go-git/go-git/v5 v5.12.0 github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 - github.com/pulumi/pulumi/sdk/v3 v3.120.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0 + github.com/pulumi/pulumi/sdk/v3 v3.121.0 github.com/spf13/cobra v1.8.1 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 @@ -29,7 +29,7 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/charmbracelet/bubbles v0.18.0 // indirect - github.com/charmbracelet/bubbletea v0.26.4 // indirect + github.com/charmbracelet/bubbletea v0.26.6 // indirect github.com/charmbracelet/lipgloss v0.11.0 // indirect github.com/charmbracelet/x/ansi v0.1.2 // indirect github.com/charmbracelet/x/input v0.1.2 // indirect @@ -68,7 +68,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.21.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -136,7 +136,7 @@ require ( golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -146,8 +146,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect k8s.io/api v0.30.2 // indirect - k8s.io/klog/v2 v2.130.0 // indirect - k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect lukechampine.com/frand v1.4.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/native-cli/go.sum b/native-cli/go.sum index b4c54ee577..8a76ee6584 100644 --- a/native-cli/go.sum +++ b/native-cli/go.sum @@ -36,8 +36,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= -github.com/charmbracelet/bubbletea v0.26.4 h1:2gDkkzLZaTjMl/dQBpNVtnvcCxsh/FCkimep7FC9c40= -github.com/charmbracelet/bubbletea v0.26.4/go.mod h1:P+r+RRA5qtI1DOHNFn0otoNwB4rn+zNAzSj/EXz6xU0= +github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= +github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g= github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8= github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= @@ -65,8 +65,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/docker v27.0.0+incompatible h1:JRugTYuelmWlW0M3jakcIadDx2HUoUO6+Tf2C5jVfwA= -github.com/docker/docker v27.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.2+incompatible h1:mNhCtgXNV1fIRns102grG7rdzIsGGCq1OlOD0KunZos= +github.com/docker/docker v27.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -146,8 +146,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= +github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -244,10 +244,10 @@ github.com/pulumi/esc v0.9.1 h1:HH5eEv8sgyxSpY5a8yePyqFXzA8cvBvapfH8457+mIs= github.com/pulumi/esc v0.9.1/go.mod h1:oEJ6bOsjYlQUpjf70GiX+CXn3VBmpwFDxUTlmtUN84c= github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 h1:plWLn9O6u80Vr37LoCsckyobBfcrdTU9cERor72QjqA= github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1/go.mod h1:N4Yu4c49QErfucPt9Y/fGmpTryRqc0VfhyKHsGR9/g8= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 h1:Fp7siNqQBjwIoY/7Jaml/v1frOyGO+kYeeMrO4d2k7k= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1/go.mod h1:MZ+ci9Iq8f0K1aOTXgD3X+ENo2+dFbgQQ7Ahh0YZ8/g= -github.com/pulumi/pulumi/sdk/v3 v3.120.0 h1:KYtMkCmcSg4U+w41/Q0l3llKEodbfdyq6J0VMoEoVmY= -github.com/pulumi/pulumi/sdk/v3 v3.120.0/go.mod h1:/mQJPO+HehhoSJ9O3C6eUKAGeAr+4KSrbDhLsXHKldc= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0 h1:y9+1n+Qh37zCronhamSmxa946T0ekM7VaJE1+UARIcM= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0/go.mod h1:Smqya4ClBt2okzTDgPkihKu/hHtoZJNkBtUWoHUUCbw= +github.com/pulumi/pulumi/sdk/v3 v3.121.0 h1:UsnFKIVOtJN/hQKPkWHL9cZktewPVQRbNUXbXQY/qrk= +github.com/pulumi/pulumi/sdk/v3 v3.121.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -415,8 +415,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -446,10 +446,10 @@ k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= -k8s.io/klog/v2 v2.130.0 h1:5nB3+3HpqKqXJIXNtJdtxcDCfaa9KL8StJgMzGJkUkM= -k8s.io/klog/v2 v2.130.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= diff --git a/native-cli/programs/local.go b/native-cli/programs/local.go index 09f2a22ab6..48520da147 100644 --- a/native-cli/programs/local.go +++ b/native-cli/programs/local.go @@ -83,6 +83,17 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { "tekton-dashboard", &components.TektonDashboard{Version: "0.45.0"}, )) + + flux, err := components.AddComponent( + ctx, + "flux", + &components.Flux{Version: "2.3.0"}, + ) + if err != nil { + log.Println(err) + os.Exit(1) + } + components.Check(components.AddComponent( ctx, "rebuild-nativelink", @@ -93,6 +104,7 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { tektonTriggers, localSources, nixStore, + flux, ), }, )) @@ -143,6 +155,17 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { }, } + capacitorGateway := components.Gateway{ + ExternalPort: 9000, //nolint:mnd + InternalPort: 9000, //nolint:mnd + Routes: []components.RouteConfig{ + { + Prefix: "/", + Cluster: "capacitor-gateway", + }, + }, + } + nativelinkGateway := components.Gateway{ ExternalPort: 8082, //nolint:mnd InternalPort: 8089, //nolint:mnd @@ -172,6 +195,7 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { "kind-loadbalancer", &components.Loadbalancer{ Gateways: []components.Gateway{ + capacitorGateway, nativelinkGateway, hubbleGateway, tknGateway, diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index f0e27c752f..8e5fd5f713 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -48,7 +48,7 @@ in { ++ [ # Integration testfiles not intended for production. "deployment-examples/docker-compose/example-do-not-use-in-prod-key.pem" - "deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem" + "kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem" ]; enable = true; name = "detect-private-key";