From 7e3fbadd3abac6419376f76431dc6a9cb9828ac2 Mon Sep 17 00:00:00 2001 From: Leonardo Milleri Date: Mon, 14 Oct 2024 11:37:55 +0100 Subject: [PATCH] kuttl integration test Run an e2e test with ephemeral kind cluster. Usage: make test-e2e The integration test is also executed as a github action Signed-off-by: Leonardo Milleri --- .github/workflows/kuttl-int-tests.yaml | 33 ++++++++++ Makefile | 10 +++ README.md | 13 ++++ kuttl-test.yaml | 5 ++ tests/e2e/sample-attester/00-assert.yaml | 7 ++ tests/e2e/sample-attester/00-install.yaml | 4 ++ tests/e2e/sample-attester/01-assert.yaml | 5 ++ tests/e2e/sample-attester/01-auth-secret.yaml | 4 ++ tests/e2e/sample-attester/02-assert.yaml | 5 ++ tests/e2e/sample-attester/02-kbs-secret.yaml | 4 ++ tests/e2e/sample-attester/03-kbs-config.yaml | 36 +++++++++++ .../sample-attester/04-resource-policy.yaml | 13 ++++ .../05-rvps-reference-values.yaml | 9 +++ tests/e2e/sample-attester/06-assert.yaml | 7 ++ .../sample-attester/06-kbsconfig_sample.yaml | 21 ++++++ tests/e2e/sample-attester/07-assert.yaml | 8 +++ tests/e2e/sample-attester/07-client.yaml | 16 +++++ tests/e2e/sample-attester/08-assert.yaml | 5 ++ tests/e2e/sample-attester/08-get-secret.yaml | 4 ++ .../e2e/sample-attester/create-auth-secret.sh | 5 ++ .../sample-attester/create-other-secret.sh | 3 + tests/e2e/sample-attester/get-secret.sh | 11 ++++ tests/e2e/sample-attester/install-operator.sh | 21 ++++++ tests/scripts/kind-with-registry.sh | 64 +++++++++++++++++++ 24 files changed, 313 insertions(+) create mode 100644 .github/workflows/kuttl-int-tests.yaml create mode 100644 kuttl-test.yaml create mode 100644 tests/e2e/sample-attester/00-assert.yaml create mode 100644 tests/e2e/sample-attester/00-install.yaml create mode 100644 tests/e2e/sample-attester/01-assert.yaml create mode 100644 tests/e2e/sample-attester/01-auth-secret.yaml create mode 100644 tests/e2e/sample-attester/02-assert.yaml create mode 100644 tests/e2e/sample-attester/02-kbs-secret.yaml create mode 100644 tests/e2e/sample-attester/03-kbs-config.yaml create mode 100644 tests/e2e/sample-attester/04-resource-policy.yaml create mode 100644 tests/e2e/sample-attester/05-rvps-reference-values.yaml create mode 100644 tests/e2e/sample-attester/06-assert.yaml create mode 100644 tests/e2e/sample-attester/06-kbsconfig_sample.yaml create mode 100644 tests/e2e/sample-attester/07-assert.yaml create mode 100644 tests/e2e/sample-attester/07-client.yaml create mode 100644 tests/e2e/sample-attester/08-assert.yaml create mode 100644 tests/e2e/sample-attester/08-get-secret.yaml create mode 100755 tests/e2e/sample-attester/create-auth-secret.sh create mode 100755 tests/e2e/sample-attester/create-other-secret.sh create mode 100755 tests/e2e/sample-attester/get-secret.sh create mode 100755 tests/e2e/sample-attester/install-operator.sh create mode 100755 tests/scripts/kind-with-registry.sh diff --git a/.github/workflows/kuttl-int-tests.yaml b/.github/workflows/kuttl-int-tests.yaml new file mode 100644 index 0000000..e20c340 --- /dev/null +++ b/.github/workflows/kuttl-int-tests.yaml @@ -0,0 +1,33 @@ +name: "Integration tests" + +on: + push: + branches: + - test-ci + pull_request: + branches: + - main  + +env: + KUTTL: /usr/local/bin/kubectl-kuttl + +jobs: + integration-tests: + runs-on: ubuntu-20.04 + steps: + + - uses: actions/setup-go@v2.1.4 + with: + go-version: 1.21 + + - uses: actions/checkout@v2.4.0 + + - name: Install dependencies + run: | + sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.19.0/kubectl-kuttl_0.19.0_linux_x86_64 + sudo chmod +x $KUTTL + sudo curl -Lo kind https://github.com/kubernetes-sigs/kind/releases/download/v0.22.0/kind-linux-amd64 + sudo chmod +x kind + + - name: "Run integration tests" + run: make test-e2e \ No newline at end of file diff --git a/Makefile b/Makefile index 4b49029..fbdb327 100644 --- a/Makefile +++ b/Makefile @@ -172,6 +172,16 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default > dist/install.yaml +# Run sample attestation in a kind cluster +# pre-requirements: kuttl plugin and kind are installed +# optional: you can change the trustee image by defining the env variable KBS_IMAGE_NAME +# Usage: KBS_IMAGE_NAME= make test-e2e +.PHONY: test-e2e +test-e2e: + ./tests/scripts/kind-with-registry.sh + kubectl kuttl test || true + kind delete cluster + ##@ Build Dependencies ## Location to install dependencies to diff --git a/README.md b/README.md index 2c6471e..135ba7c 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,19 @@ which provide a reconcile function responsible for synchronizing resources until **NOTE:** You can also run this in one step by running: `make install run` +### Integration tests + +An attestation with the sample-attester is performed in an ephemeral kind cluster +Pre-requirements: +- [kuttl](https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin) plugin installed +- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) installed + +Optional: set the env variable KBS_IMAGE_NAME to override the default trustee image + + ```sh + KBS_IMAGE_NAME= make test-e2e + ``` + ### Modifying the API definitions If you are editing the API definitions, generate the manifests such as CRs or CRDs using: diff --git a/kuttl-test.yaml b/kuttl-test.yaml new file mode 100644 index 0000000..42f364d --- /dev/null +++ b/kuttl-test.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- tests/e2e/ +timeout: 240 diff --git a/tests/e2e/sample-attester/00-assert.yaml b/tests/e2e/sample-attester/00-assert.yaml new file mode 100644 index 0000000..f497483 --- /dev/null +++ b/tests/e2e/sample-attester/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trustee-operator-controller-manager + namespace: trustee-operator-system +status: + readyReplicas: 1 diff --git a/tests/e2e/sample-attester/00-install.yaml b/tests/e2e/sample-attester/00-install.yaml new file mode 100644 index 0000000..430c232 --- /dev/null +++ b/tests/e2e/sample-attester/00-install.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: ./install-operator.sh diff --git a/tests/e2e/sample-attester/01-assert.yaml b/tests/e2e/sample-attester/01-assert.yaml new file mode 100644 index 0000000..94c06fe --- /dev/null +++ b/tests/e2e/sample-attester/01-assert.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kbs-auth-public-key + namespace: trustee-operator-system diff --git a/tests/e2e/sample-attester/01-auth-secret.yaml b/tests/e2e/sample-attester/01-auth-secret.yaml new file mode 100644 index 0000000..6810685 --- /dev/null +++ b/tests/e2e/sample-attester/01-auth-secret.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: ./create-auth-secret.sh diff --git a/tests/e2e/sample-attester/02-assert.yaml b/tests/e2e/sample-attester/02-assert.yaml new file mode 100644 index 0000000..bcb791a --- /dev/null +++ b/tests/e2e/sample-attester/02-assert.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kbsres1 + namespace: trustee-operator-system diff --git a/tests/e2e/sample-attester/02-kbs-secret.yaml b/tests/e2e/sample-attester/02-kbs-secret.yaml new file mode 100644 index 0000000..ee09837 --- /dev/null +++ b/tests/e2e/sample-attester/02-kbs-secret.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: ./create-other-secret.sh diff --git a/tests/e2e/sample-attester/03-kbs-config.yaml b/tests/e2e/sample-attester/03-kbs-config.yaml new file mode 100644 index 0000000..2578399 --- /dev/null +++ b/tests/e2e/sample-attester/03-kbs-config.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kbs-config + namespace: trustee-operator-system +data: + kbs-config.json: | + { + "insecure_http" : true, + "sockets": ["0.0.0.0:8080"], + "auth_public_key": "/etc/auth-secret/publicKey", + "attestation_token_config": { + "attestation_token_type": "CoCo" + }, + "repository_config": { + "type": "LocalFs", + "dir_path": "/opt/confidential-containers/kbs/repository" + }, + "as_config": { + "work_dir": "/opt/confidential-containers/attestation-service", + "policy_engine": "opa", + "attestation_token_broker": "Simple", + "attestation_token_config": { + "duration_min": 5 + }, + "rvps_config": { + "store_type": "LocalJson", + "store_config": { + "file_path": "/opt/confidential-containers/rvps/reference-values/reference-values.json" + } + } + }, + "policy_engine_config": { + "policy_path": "/opt/confidential-containers/opa/policy.rego" + } + } diff --git a/tests/e2e/sample-attester/04-resource-policy.yaml b/tests/e2e/sample-attester/04-resource-policy.yaml new file mode 100644 index 0000000..b6267d1 --- /dev/null +++ b/tests/e2e/sample-attester/04-resource-policy.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: resource-policy + namespace: trustee-operator-system +data: + policy.rego: | + package policy + + default allow = false + allow { + input["tcb-status"]["sample.svn"] == "1" + } diff --git a/tests/e2e/sample-attester/05-rvps-reference-values.yaml b/tests/e2e/sample-attester/05-rvps-reference-values.yaml new file mode 100644 index 0000000..2eade40 --- /dev/null +++ b/tests/e2e/sample-attester/05-rvps-reference-values.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rvps-reference-values + namespace: trustee-operator-system +data: + reference-values.json: | + [ + ] diff --git a/tests/e2e/sample-attester/06-assert.yaml b/tests/e2e/sample-attester/06-assert.yaml new file mode 100644 index 0000000..0513c8a --- /dev/null +++ b/tests/e2e/sample-attester/06-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trustee-deployment + namespace: trustee-operator-system +status: + readyReplicas: 1 \ No newline at end of file diff --git a/tests/e2e/sample-attester/06-kbsconfig_sample.yaml b/tests/e2e/sample-attester/06-kbsconfig_sample.yaml new file mode 100644 index 0000000..5a812a4 --- /dev/null +++ b/tests/e2e/sample-attester/06-kbsconfig_sample.yaml @@ -0,0 +1,21 @@ +apiVersion: confidentialcontainers.org/v1alpha1 +kind: KbsConfig +metadata: + labels: + app.kubernetes.io/name: kbsconfig + app.kubernetes.io/instance: kbsconfig-sample + app.kubernetes.io/part-of: trustee-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: trustee-operator + name: kbsconfig-sample + namespace: trustee-operator-system +spec: + kbsConfigMapName: kbs-config + kbsAuthSecretName: kbs-auth-public-key + kbsDeploymentType: AllInOneDeployment + kbsRvpsRefValuesConfigMapName: rvps-reference-values + kbsResourcePolicyConfigMapName: resource-policy + kbsSecretResources: + - "kbsres1" + KbsEnvVars: + RUST_LOG: debug \ No newline at end of file diff --git a/tests/e2e/sample-attester/07-assert.yaml b/tests/e2e/sample-attester/07-assert.yaml new file mode 100644 index 0000000..6ae73a3 --- /dev/null +++ b/tests/e2e/sample-attester/07-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kbs-client + namespace: trustee-operator-system +status: + containerStatuses: + - ready: true \ No newline at end of file diff --git a/tests/e2e/sample-attester/07-client.yaml b/tests/e2e/sample-attester/07-client.yaml new file mode 100644 index 0000000..4db5086 --- /dev/null +++ b/tests/e2e/sample-attester/07-client.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kbs-client + namespace: trustee-operator-system +spec: + containers: + - name: kbs-client + image: quay.io/confidential-containers/kbs-client:v0.9.0 + imagePullPolicy: IfNotPresent + command: + - sleep + - "360000" + env: + - name: RUST_LOG + value: none diff --git a/tests/e2e/sample-attester/08-assert.yaml b/tests/e2e/sample-attester/08-assert.yaml new file mode 100644 index 0000000..40cc627 --- /dev/null +++ b/tests/e2e/sample-attester/08-assert.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: trustee-secret + namespace: trustee-operator-system diff --git a/tests/e2e/sample-attester/08-get-secret.yaml b/tests/e2e/sample-attester/08-get-secret.yaml new file mode 100644 index 0000000..a4bba19 --- /dev/null +++ b/tests/e2e/sample-attester/08-get-secret.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: ./get-secret.sh diff --git a/tests/e2e/sample-attester/create-auth-secret.sh b/tests/e2e/sample-attester/create-auth-secret.sh new file mode 100755 index 0000000..782a911 --- /dev/null +++ b/tests/e2e/sample-attester/create-auth-secret.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +openssl genpkey -algorithm ed25519 > privateKey +openssl pkey -in privateKey -pubout -out publicKey +kubectl create secret generic kbs-auth-public-key --from-file=publicKey -n trustee-operator-system \ No newline at end of file diff --git a/tests/e2e/sample-attester/create-other-secret.sh b/tests/e2e/sample-attester/create-other-secret.sh new file mode 100755 index 0000000..9fdb980 --- /dev/null +++ b/tests/e2e/sample-attester/create-other-secret.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl create secret generic kbsres1 --from-literal key1=res1val1 --from-literal key2=res1val2 -n trustee-operator-system \ No newline at end of file diff --git a/tests/e2e/sample-attester/get-secret.sh b/tests/e2e/sample-attester/get-secret.sh new file mode 100755 index 0000000..b1c9b13 --- /dev/null +++ b/tests/e2e/sample-attester/get-secret.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +SECRET="$(kubectl exec -it -n trustee-operator-system kbs-client -- kbs-client --url http://kbs-service:8080 get-resource --path default/kbsres1/key1)" +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Error when retrieving the secret" +else + echo "Attestation completed successfully: secret="$SECRET + # this secret is created only to check the secret retrieval has been successful + kubectl create secret generic trustee-secret --from-literal key1=$SECRET -n trustee-operator-system +fi diff --git a/tests/e2e/sample-attester/install-operator.sh b/tests/e2e/sample-attester/install-operator.sh new file mode 100755 index 0000000..45cc809 --- /dev/null +++ b/tests/e2e/sample-attester/install-operator.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +KBS_IMAGE_NAME="${KBS_IMAGE_NAME:-quay.io/confidential-containers/trustee:290fd0eb64ab20f50efbd27cf80542851c0ee17f}" +REGISTRY=localhost:5001 +export IMG=${REGISTRY}/trustee-operator:test + +pushd ../../.. + +pushd config/manager +kustomize edit set image controller=$IMG +kustomize edit add patch --patch "- op: replace + path: '/spec/template/spec/containers/0/env/1' + value: + name: KBS_IMAGE_NAME + value: ${KBS_IMAGE_NAME}" --kind Deployment --name controller-manager +popd +make docker-build docker-push +make build-installer +kubectl apply -f dist/install.yaml + +popd diff --git a/tests/scripts/kind-with-registry.sh b/tests/scripts/kind-with-registry.sh new file mode 100755 index 0000000..9fe55a8 --- /dev/null +++ b/tests/scripts/kind-with-registry.sh @@ -0,0 +1,64 @@ +#!/bin/sh +set -o errexit + +# 1. Create registry container unless it already exists +reg_name='kind-registry' +reg_port='5001' +if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \ + registry:2 +fi + +# 2. Create kind cluster with containerd registry config dir enabled +# TODO: kind will eventually enable this by default and this patch will +# be unnecessary. +# +# See: +# https://github.com/kubernetes-sigs/kind/issues/2875 +# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration +# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md +cat <