From c2637ead30a4076e11f81ce0f9b6d81732856e62 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Fri, 13 Jan 2023 11:21:35 +0100 Subject: [PATCH] templates: add flatcar templates It mainly drifts from other templates by using Ignition bootstrap experimental feature. Signed-off-by: Mathieu Tortuyaux --- Makefile | 14 +- hack/ci/cloud-init/controller.yaml.tpl | 5 +- .../kustomization.yaml | 6 + .../patch-ccm.yaml | 43 ++++ .../patch-flatcar.yaml | 97 ++++++++ kustomize/v1alpha6/flatcar/kustomization.yaml | 5 + kustomize/v1alpha6/flatcar/patch-flatcar.yaml | 117 +++++++++ ...plate-external-cloud-provider-flatcar.yaml | 189 +++++++++++++++ templates/cluster-template-flatcar.yaml | 228 ++++++++++++++++++ test/e2e/data/e2e_conf.yaml | 4 + .../.gitignore | 1 + .../common-patches/{ => cni}/cni.yaml | 0 .../common-patches/cni/kustomization.yaml | 16 ++ .../{ => cni}/patch-cluster.yaml | 0 .../common-patches/{ => cni}/patch-cni.yaml | 0 .../{ => containerd}/containerd-kcp.yaml | 0 .../{ => containerd}/containerd-kct.yaml | 0 .../{ => containerd}/kustomization.yaml | 10 - .../data/kustomize/default/kustomization.yaml | 3 +- .../external-cloud-provider-flatcar/ccm.yaml | 19 ++ .../kustomization.yaml | 20 ++ .../patch-allow-all-in-cluster-traffic.yaml | 4 + .../patch-ccm-cloud-config.yaml | 16 ++ .../patch-ccm.yaml | 7 + .../kustomization.yaml | 3 +- .../data/kustomize/flatcar/kustomization.yaml | 5 + .../kustomize/v1alpha5/kustomization.yaml | 3 +- .../kustomize/v1alpha6/kustomization.yaml | 3 +- .../kustomize/without-lb/kustomization.yaml | 3 +- test/e2e/shared/common.go | 3 + test/e2e/shared/defaults.go | 47 ++-- test/e2e/shared/exec.go | 60 +++-- test/e2e/shared/exec_test.go | 2 +- test/e2e/shared/suite.go | 41 +++- test/e2e/suites/e2e/e2e_test.go | 73 ++++++ 35 files changed, 979 insertions(+), 68 deletions(-) create mode 100644 kustomize/v1alpha6/external-cloud-provider-flatcar/kustomization.yaml create mode 100644 kustomize/v1alpha6/external-cloud-provider-flatcar/patch-ccm.yaml create mode 100644 kustomize/v1alpha6/external-cloud-provider-flatcar/patch-flatcar.yaml create mode 100644 kustomize/v1alpha6/flatcar/kustomization.yaml create mode 100644 kustomize/v1alpha6/flatcar/patch-flatcar.yaml create mode 100644 templates/cluster-template-external-cloud-provider-flatcar.yaml create mode 100644 templates/cluster-template-flatcar.yaml create mode 100644 test/e2e/data/infrastructure-openstack-no-artifact/.gitignore rename test/e2e/data/kustomize/common-patches/{ => cni}/cni.yaml (100%) create mode 100644 test/e2e/data/kustomize/common-patches/cni/kustomization.yaml rename test/e2e/data/kustomize/common-patches/{ => cni}/patch-cluster.yaml (100%) rename test/e2e/data/kustomize/common-patches/{ => cni}/patch-cni.yaml (100%) rename test/e2e/data/kustomize/common-patches/{ => containerd}/containerd-kcp.yaml (100%) rename test/e2e/data/kustomize/common-patches/{ => containerd}/containerd-kct.yaml (100%) rename test/e2e/data/kustomize/common-patches/{ => containerd}/kustomization.yaml (69%) create mode 100644 test/e2e/data/kustomize/external-cloud-provider-flatcar/ccm.yaml create mode 100644 test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml create mode 100644 test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-allow-all-in-cluster-traffic.yaml create mode 100644 test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm-cloud-config.yaml create mode 100644 test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm.yaml create mode 100644 test/e2e/data/kustomize/flatcar/kustomization.yaml diff --git a/Makefile b/Makefile index 2efdbab87d..2fdb3fb388 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,8 @@ test: $(SETUP_ENVTEST) ## Run tests E2E_TEMPLATES_DIR=test/e2e/data/infrastructure-openstack E2E_KUSTOMIZE_DIR=test/e2e/data/kustomize +# This directory holds the templates that do not require ci-artifacts script injection. +E2E_NO_ARTIFACT_TEMPLATES_DIR=test/e2e/data/infrastructure-openstack-no-artifact .PHONY: e2e-templates e2e-templates: ## Generate cluster templates for e2e tests @@ -159,7 +161,10 @@ e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \ cluster-template-multi-az.yaml \ cluster-template-multi-network.yaml \ cluster-template-without-lb.yaml \ - cluster-template.yaml) + cluster-template.yaml) \ + $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \ + cluster-template-flatcar.yaml \ + cluster-template-external-cloud-provider-flatcar.yaml) \ $(E2E_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE $(KUSTOMIZE) build "$<" > "$@" @@ -167,6 +172,9 @@ $(E2E_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUS $(E2E_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE $(KUSTOMIZE) build "$<" > "$@" +$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE + $(KUSTOMIZE) build "$<" > "$@" + e2e-prerequisites: $(GINKGO) e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests # Can be run manually, e.g. via: @@ -417,7 +425,9 @@ release-notes: $(RELEASE_NOTES) ## Generate release notes templates: ## Generate cluster templates templates: templates/cluster-template.yaml \ templates/cluster-template-without-lb.yaml \ - templates/cluster-template-external-cloud-provider.yaml + templates/cluster-template-external-cloud-provider.yaml \ + templates/cluster-template-flatcar.yaml \ + templates/cluster-template-external-cloud-provider-flatcar.yaml \ templates/cluster-template.yaml: kustomize/v1alpha7/default $(KUSTOMIZE) FORCE $(KUSTOMIZE) build "$<" > "$@" diff --git a/hack/ci/cloud-init/controller.yaml.tpl b/hack/ci/cloud-init/controller.yaml.tpl index 6ee7648234..5ce777cfbb 100644 --- a/hack/ci/cloud-init/controller.yaml.tpl +++ b/hack/ci/cloud-init/controller.yaml.tpl @@ -138,6 +138,7 @@ # https://docs.openstack.org/glance/latest/admin/quotas.html /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-01-14/focal-server-cloudimg-amd64.img + /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3374.2.5-kube-v1.25.6.img # Add the controller to its own host aggregate and availability zone aggregateid=$(openstack aggregate create --zone "${PRIMARY_AZ}" "${PRIMARY_AZ}" -f value -c id) @@ -150,9 +151,9 @@ openstack flavor delete m1.tiny openstack flavor create --ram 512 --disk 1 --vcpus 1 --public --id 1 m1.tiny --property hw_rng:allowed='True' openstack flavor delete m1.small - openstack flavor create --ram 4192 --disk 10 --vcpus 2 --public --id 2 m1.small --property hw_rng:allowed='True' + openstack flavor create --ram 4192 --disk 20 --vcpus 2 --public --id 2 m1.small --property hw_rng:allowed='True' openstack flavor delete m1.medium - openstack flavor create --ram 6144 --disk 10 --vcpus 4 --public --id 3 m1.medium --property hw_rng:allowed='True' + openstack flavor create --ram 6144 --disk 20 --vcpus 4 --public --id 3 m1.medium --property hw_rng:allowed='True' # Adjust the CPU quota openstack quota set --cores 32 demo diff --git a/kustomize/v1alpha6/external-cloud-provider-flatcar/kustomization.yaml b/kustomize/v1alpha6/external-cloud-provider-flatcar/kustomization.yaml new file mode 100644 index 0000000000..cea562938e --- /dev/null +++ b/kustomize/v1alpha6/external-cloud-provider-flatcar/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- ../default + +patchesStrategicMerge: +- patch-ccm.yaml +- patch-flatcar.yaml diff --git a/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-ccm.yaml b/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-ccm.yaml new file mode 100644 index 0000000000..35ba1cdaf7 --- /dev/null +++ b/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-ccm.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + cloud-config: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + cloud-config: + extraVolumes: + controllerManager: + extraArgs: + cloud-provider: external + cloud-config: + extraVolumes: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + cloud-config: + files: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: [] + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + cloud-config: diff --git a/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-flatcar.yaml b/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-flatcar.yaml new file mode 100644 index 0000000000..d6de831a3f --- /dev/null +++ b/kustomize/v1alpha6/external-cloud-provider-flatcar/patch-flatcar.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + apiServerLoadBalancer: + $patch: delete +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + kubeadmConfigSpec: + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + initConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} diff --git a/kustomize/v1alpha6/flatcar/kustomization.yaml b/kustomize/v1alpha6/flatcar/kustomization.yaml new file mode 100644 index 0000000000..1ad5cb6720 --- /dev/null +++ b/kustomize/v1alpha6/flatcar/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- ../default + +patchesStrategicMerge: +- patch-flatcar.yaml diff --git a/kustomize/v1alpha6/flatcar/patch-flatcar.yaml b/kustomize/v1alpha6/flatcar/patch-flatcar.yaml new file mode 100644 index 0000000000..30135c37d6 --- /dev/null +++ b/kustomize/v1alpha6/flatcar/patch-flatcar.yaml @@ -0,0 +1,117 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + apiServerLoadBalancer: + $patch: delete +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + kubeadmConfigSpec: + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + initConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + cloud-provider: openstack + cloud-config: /etc/kubernetes/cloud.conf + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" + - content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} diff --git a/templates/cluster-template-external-cloud-provider-flatcar.yaml b/templates/cluster-template-external-cloud-provider-flatcar.yaml new file mode 100644 index 0000000000..9162eb92dd --- /dev/null +++ b/templates/cluster-template-external-cloud-provider-flatcar.yaml @@ -0,0 +1,189 @@ +apiVersion: v1 +data: + cacert: ${OPENSTACK_CLOUD_CACERT_B64} + clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64} +kind: Secret +metadata: + labels: + clusterctl.cluster.x-k8s.io/move: "true" + name: ${CLUSTER_NAME}-cloud-config +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: [] + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + failureDomain: ${OPENSTACK_FAILURE_DOMAIN} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + files: [] + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + cloudName: ${OPENSTACK_CLOUD} + dnsNameservers: + - ${OPENSTACK_DNS_NAMESERVERS} + externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + managedSecurityGroups: true + nodeCidr: 10.6.0.0/24 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml new file mode 100644 index 0000000000..46ab7623c4 --- /dev/null +++ b/templates/cluster-template-flatcar.yaml @@ -0,0 +1,228 @@ +apiVersion: v1 +data: + cacert: ${OPENSTACK_CLOUD_CACERT_B64} + clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64} +kind: Secret +metadata: + labels: + clusterctl.cluster.x-k8s.io/move: "true" + name: ${CLUSTER_NAME}-cloud-config +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" + - content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + failureDomain: ${OPENSTACK_FAILURE_DOMAIN} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + extraVolumes: + - hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + name: cloud + readOnly: true + controllerManager: + extraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + extraVolumes: + - hostPath: /etc/kubernetes/cloud.conf + mountPath: /etc/kubernetes/cloud.conf + name: cloud + readOnly: true + - hostPath: /etc/certs/cacert + mountPath: /etc/certs/cacert + name: cacerts + readOnly: true + files: + - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" + - content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + + [Service] + EnvironmentFile=/run/metadata/flatcar + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + name: $${COREOS_OPENSTACK_HOSTNAME} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/cloud.conf + cloud-provider: openstack + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + cloudName: ${OPENSTACK_CLOUD} + dnsNameservers: + - ${OPENSTACK_DNS_NAMESERVERS} + externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + managedSecurityGroups: true + nodeCidr: 10.6.0.0/24 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index be6f03f812..cb1d405d30 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -156,6 +156,10 @@ variables: CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1" INIT_WITH_KUBERNETES_VERSION: "v1.25.0" E2E_IMAGE_URL: "http://10.0.3.15/capo-e2e-image.tar" + # The default user for SSH connections from bastion to machines + SSH_USER_MACHINE: "ubuntu" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" + OPENSTACK_FLATCAR_IMAGE_NAME: "flatcar-stable-3374.2.5-kube-v1.25.6" intervals: conformance/wait-control-plane: ["30m", "10s"] diff --git a/test/e2e/data/infrastructure-openstack-no-artifact/.gitignore b/test/e2e/data/infrastructure-openstack-no-artifact/.gitignore new file mode 100644 index 0000000000..4beb15c8a2 --- /dev/null +++ b/test/e2e/data/infrastructure-openstack-no-artifact/.gitignore @@ -0,0 +1 @@ +/cluster-template*.yaml diff --git a/test/e2e/data/kustomize/common-patches/cni.yaml b/test/e2e/data/kustomize/common-patches/cni/cni.yaml similarity index 100% rename from test/e2e/data/kustomize/common-patches/cni.yaml rename to test/e2e/data/kustomize/common-patches/cni/cni.yaml diff --git a/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml b/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml new file mode 100644 index 0000000000..d01d1cb1e8 --- /dev/null +++ b/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml @@ -0,0 +1,16 @@ +# Modifications to release templates common to all e2e test scenarios +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- cni.yaml + +patchesStrategicMerge: +- patch-cni.yaml + +patches: +- target: + kind: OpenStackCluster + name: \${CLUSTER_NAME} + path: patch-cluster.yaml diff --git a/test/e2e/data/kustomize/common-patches/patch-cluster.yaml b/test/e2e/data/kustomize/common-patches/cni/patch-cluster.yaml similarity index 100% rename from test/e2e/data/kustomize/common-patches/patch-cluster.yaml rename to test/e2e/data/kustomize/common-patches/cni/patch-cluster.yaml diff --git a/test/e2e/data/kustomize/common-patches/patch-cni.yaml b/test/e2e/data/kustomize/common-patches/cni/patch-cni.yaml similarity index 100% rename from test/e2e/data/kustomize/common-patches/patch-cni.yaml rename to test/e2e/data/kustomize/common-patches/cni/patch-cni.yaml diff --git a/test/e2e/data/kustomize/common-patches/containerd-kcp.yaml b/test/e2e/data/kustomize/common-patches/containerd/containerd-kcp.yaml similarity index 100% rename from test/e2e/data/kustomize/common-patches/containerd-kcp.yaml rename to test/e2e/data/kustomize/common-patches/containerd/containerd-kcp.yaml diff --git a/test/e2e/data/kustomize/common-patches/containerd-kct.yaml b/test/e2e/data/kustomize/common-patches/containerd/containerd-kct.yaml similarity index 100% rename from test/e2e/data/kustomize/common-patches/containerd-kct.yaml rename to test/e2e/data/kustomize/common-patches/containerd/containerd-kct.yaml diff --git a/test/e2e/data/kustomize/common-patches/kustomization.yaml b/test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml similarity index 69% rename from test/e2e/data/kustomize/common-patches/kustomization.yaml rename to test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml index ac018312e8..1750875bc8 100644 --- a/test/e2e/data/kustomize/common-patches/kustomization.yaml +++ b/test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml @@ -3,17 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component -resources: -- cni.yaml - -patchesStrategicMerge: -- patch-cni.yaml - patches: -- target: - kind: OpenStackCluster - name: \${CLUSTER_NAME} - path: patch-cluster.yaml - target: kind: KubeadmControlPlane name: \${CLUSTER_NAME}-control-plane diff --git a/test/e2e/data/kustomize/default/kustomization.yaml b/test/e2e/data/kustomize/default/kustomization.yaml index 6e0df86896..9db6793d1e 100644 --- a/test/e2e/data/kustomize/default/kustomization.yaml +++ b/test/e2e/data/kustomize/default/kustomization.yaml @@ -3,5 +3,6 @@ resources: - ../../../../../kustomize/v1alpha7/default components: -- ../common-patches +- ../common-patches/cni +- ../common-patches/containerd - ../upgrade-patches diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/ccm.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/ccm.yaml new file mode 100644 index 0000000000..a06a280498 --- /dev/null +++ b/test/e2e/data/kustomize/external-cloud-provider-flatcar/ccm.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccm-${CLUSTER_NAME}-crs-1 +data: ${CCM_RESOURCES} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-1 +spec: + clusterSelector: + matchLabels: + ccm: ${CLUSTER_NAME}-crs-1 + resources: + - kind: ConfigMap + name: ccm-${CLUSTER_NAME}-crs-1 + strategy: ApplyOnce diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml new file mode 100644 index 0000000000..6051c4a8a4 --- /dev/null +++ b/test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml @@ -0,0 +1,20 @@ +--- +resources: +- ../../../../../kustomize/v1alpha6/external-cloud-provider-flatcar +- ccm.yaml + +components: +- ../common-patches/cni + +patchesStrategicMerge: +- patch-ccm.yaml + +patches: +- target: + kind: OpenStackCluster + name: \${CLUSTER_NAME} + path: patch-allow-all-in-cluster-traffic.yaml +- target: + kind: KubeadmControlPlane + name: \${CLUSTER_NAME}-control-plane + path: patch-ccm-cloud-config.yaml diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-allow-all-in-cluster-traffic.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-allow-all-in-cluster-traffic.yaml new file mode 100644 index 0000000000..240bc43cd0 --- /dev/null +++ b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-allow-all-in-cluster-traffic.yaml @@ -0,0 +1,4 @@ +--- +- op: add + path: /spec/allowAllInClusterTraffic + value: true diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm-cloud-config.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm-cloud-config.yaml new file mode 100644 index 0000000000..fb537ce436 --- /dev/null +++ b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm-cloud-config.yaml @@ -0,0 +1,16 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + owner: root + path: /etc/kubernetes/cloud.conf + permissions: "0600" +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 + owner: root + path: /etc/certs/cacert + permissions: "0600" diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm.yaml new file mode 100644 index 0000000000..9d3556597f --- /dev/null +++ b/test/e2e/data/kustomize/external-cloud-provider-flatcar/patch-ccm.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + ccm: ${CLUSTER_NAME}-crs-1 diff --git a/test/e2e/data/kustomize/external-cloud-provider/kustomization.yaml b/test/e2e/data/kustomize/external-cloud-provider/kustomization.yaml index 365dcb5208..2d35cd215a 100644 --- a/test/e2e/data/kustomize/external-cloud-provider/kustomization.yaml +++ b/test/e2e/data/kustomize/external-cloud-provider/kustomization.yaml @@ -4,7 +4,8 @@ resources: - ccm.yaml components: -- ../common-patches +- ../common-patches/cni +- ../common-patches/containerd patchesStrategicMerge: - patch-ccm.yaml diff --git a/test/e2e/data/kustomize/flatcar/kustomization.yaml b/test/e2e/data/kustomize/flatcar/kustomization.yaml new file mode 100644 index 0000000000..dd5ef59a49 --- /dev/null +++ b/test/e2e/data/kustomize/flatcar/kustomization.yaml @@ -0,0 +1,5 @@ +--- +resources: +- ../../../../../kustomize/v1alpha6/flatcar +components: +- ../common-patches/cni diff --git a/test/e2e/data/kustomize/v1alpha5/kustomization.yaml b/test/e2e/data/kustomize/v1alpha5/kustomization.yaml index 4a4b95323c..716418425b 100644 --- a/test/e2e/data/kustomize/v1alpha5/kustomization.yaml +++ b/test/e2e/data/kustomize/v1alpha5/kustomization.yaml @@ -3,4 +3,5 @@ resources: - ../../../../../kustomize/v1alpha5/default components: -- ../common-patches +- ../common-patches/cni +- ../common-patches/containerd diff --git a/test/e2e/data/kustomize/v1alpha6/kustomization.yaml b/test/e2e/data/kustomize/v1alpha6/kustomization.yaml index 8609199ac8..cd50d80ae5 100644 --- a/test/e2e/data/kustomize/v1alpha6/kustomization.yaml +++ b/test/e2e/data/kustomize/v1alpha6/kustomization.yaml @@ -3,4 +3,5 @@ resources: - ../../../../../kustomize/v1alpha6/default components: -- ../common-patches +- ../common-patches/cni +- ../common-patches/containerd diff --git a/test/e2e/data/kustomize/without-lb/kustomization.yaml b/test/e2e/data/kustomize/without-lb/kustomization.yaml index ee884d104d..eccccda282 100644 --- a/test/e2e/data/kustomize/without-lb/kustomization.yaml +++ b/test/e2e/data/kustomize/without-lb/kustomization.yaml @@ -3,4 +3,5 @@ resources: - ../../../../../kustomize/v1alpha7/without-lb components: -- ../common-patches +- ../common-patches/cni +- ../common-patches/containerd diff --git a/test/e2e/shared/common.go b/test/e2e/shared/common.go index b94bc9448a..2f92e22774 100644 --- a/test/e2e/shared/common.go +++ b/test/e2e/shared/common.go @@ -176,6 +176,8 @@ func dumpMachine(ctx context.Context, e2eCtx *E2EContext, machine infrav1.OpenSt _, _ = fmt.Fprintf(GinkgoWriter, "error writing server JSON %s: %s", serverJSON, err) } + srvUser := e2eCtx.E2EConfig.GetVariable(SSHUserMachine) + _, _ = fmt.Fprintf(f, "instance found: %q\n", srv.ID) executeCommands( ctx, @@ -184,6 +186,7 @@ func dumpMachine(ctx context.Context, e2eCtx *E2EContext, machine infrav1.OpenSt filepath.Dir(f.Name()), srv.ip, bastionIP, + srvUser, []command{ // don't do this for now, it just takes to long // { diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 55080263b2..e5e405d20b 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -30,28 +30,31 @@ import ( ) const ( - DefaultSSHKeyPairName = "cluster-api-provider-openstack-sigs-k8s-io" - KubeContext = "KUBE_CONTEXT" - KubernetesVersion = "KUBERNETES_VERSION" - CCMPath = "CCM" - CCMResources = "CCM_RESOURCES" - OpenStackCloudYAMLFile = "OPENSTACK_CLOUD_YAML_FILE" - OpenStackCloud = "OPENSTACK_CLOUD" - OpenStackCloudAdmin = "OPENSTACK_CLOUD_ADMIN" - OpenStackFailureDomain = "OPENSTACK_FAILURE_DOMAIN" - OpenStackFailureDomainAlt = "OPENSTACK_FAILURE_DOMAIN_ALT" - OpenStackVolumeTypeAlt = "OPENSTACK_VOLUME_TYPE_ALT" - OpenStackImageName = "OPENSTACK_IMAGE_NAME" - OpenStackNodeMachineFlavor = "OPENSTACK_NODE_MACHINE_FLAVOR" - FlavorDefault = "ci-artifacts" - FlavorWithoutLB = "without-lb-ci-artifacts" - FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts" - FlavorMultiNetwork = "multi-network-ci-artifacts" - FlavorMultiAZ = "multi-az-ci-artifacts" - FlavorV1alpha5 = "v1alpha5-ci-artifacts" - FlavorV1alpha6 = "v1alpha6-ci-artifacts" - FlavorMDRemediation = "md-remediation-ci-artifacts" - FlavorKCPRemediation = "kcp-remediation-ci-artifacts" + DefaultSSHKeyPairName = "cluster-api-provider-openstack-sigs-k8s-io" + KubeContext = "KUBE_CONTEXT" + KubernetesVersion = "KUBERNETES_VERSION" + CCMPath = "CCM" + CCMResources = "CCM_RESOURCES" + OpenStackCloudYAMLFile = "OPENSTACK_CLOUD_YAML_FILE" + OpenStackCloud = "OPENSTACK_CLOUD" + OpenStackCloudAdmin = "OPENSTACK_CLOUD_ADMIN" + OpenStackFailureDomain = "OPENSTACK_FAILURE_DOMAIN" + OpenStackFailureDomainAlt = "OPENSTACK_FAILURE_DOMAIN_ALT" + OpenStackVolumeTypeAlt = "OPENSTACK_VOLUME_TYPE_ALT" + OpenStackImageName = "OPENSTACK_IMAGE_NAME" + OpenStackNodeMachineFlavor = "OPENSTACK_NODE_MACHINE_FLAVOR" + SSHUserMachine = "SSH_USER_MACHINE" + FlavorDefault = "ci-artifacts" + FlavorWithoutLB = "without-lb-ci-artifacts" + FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts" + FlavorMultiNetwork = "multi-network-ci-artifacts" + FlavorMultiAZ = "multi-az-ci-artifacts" + FlavorV1alpha5 = "v1alpha5-ci-artifacts" + FlavorV1alpha6 = "v1alpha6-ci-artifacts" + FlavorMDRemediation = "md-remediation-ci-artifacts" + FlavorKCPRemediation = "kcp-remediation-ci-artifacts" + FlavorFlatcar = "flatcar" + FlavorExternalCloudProviderFlatcar = "external-cloud-provider-flatcar" ) // DefaultScheme returns the default scheme to use for testing. diff --git a/test/e2e/shared/exec.go b/test/e2e/shared/exec.go index 8141f957ee..df85fcbdd0 100644 --- a/test/e2e/shared/exec.go +++ b/test/e2e/shared/exec.go @@ -39,9 +39,19 @@ type command struct { cmd string } +type commandParameter struct { + signer ssh.Signer + debug bool + logDir string + bastionIP string + machineIP string + machineUser string + cmd command +} + // executeCommands opens a terminal connection // and executes the given commands, outputting the results to a file for each. -func executeCommands(ctx context.Context, artifactsFolder string, debug bool, logDir, machineIP, bastionIP string, commands []command) { +func executeCommands(ctx context.Context, artifactsFolder string, debug bool, logDir, machineIP, bastionIP, machineUser string, commands []command) { privateKey, err := os.ReadFile(filepath.Join(artifactsFolder, "ssh", DefaultSSHKeyPairName)) if err != nil { _, _ = fmt.Fprintf(GinkgoWriter, "could not load private key from artifacts folder: %s\n", err) @@ -54,31 +64,39 @@ func executeCommands(ctx context.Context, artifactsFolder string, debug bool, lo } for _, cmd := range commands { - if err := executeCommand(ctx, signer, debug, logDir, bastionIP, machineIP, cmd); err != nil { + if err := executeCommand(ctx, commandParameter{ + signer: signer, + debug: debug, + logDir: logDir, + bastionIP: bastionIP, + machineIP: machineIP, + machineUser: machineUser, + cmd: cmd, + }); err != nil { _, _ = fmt.Fprintln(GinkgoWriter, err.Error()) } } } -func executeCommand(ctx context.Context, signer ssh.Signer, debug bool, logDir, bastionIP, machineIP string, cmd command) error { +func executeCommand(ctx context.Context, p commandParameter) error { cfg := &ssh.ClientConfig{ User: "cirros", Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), + ssh.PublicKeys(p.signer), }, HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }, Timeout: 60 * time.Second, } cfg.SetDefaults() - Debugf(debug, "dialing from local to bastion host %s", bastionIP) - bastionConn, err := ssh.Dial("tcp", fmt.Sprintf("%s:22", bastionIP), cfg) + Debugf(p.debug, "dialing from local to bastion host %s", p.bastionIP) + bastionConn, err := ssh.Dial("tcp", fmt.Sprintf("%s:22", p.bastionIP), cfg) if err != nil { - return fmt.Errorf("couldn't dial from local to bastion host %s: %s", bastionIP, err) + return fmt.Errorf("couldn't dial from local to bastion host %s: %s", p.bastionIP, err) } defer bastionConn.Close() // Dial a connection to the service host, from the bastion host - Debugf(debug, "dialing from bastion host %s to machine %s", bastionIP, machineIP) + Debugf(p.debug, "dialing from bastion host %s to machine %s", p.bastionIP, p.machineIP) // we have to timeout this connection // * there is no way to set a timeout in the Dial func // * sometimes the server are deleted when we try this and we would be stuck infinitely @@ -88,50 +106,50 @@ func executeCommand(ctx context.Context, signer ssh.Signer, debug bool, logDir, <-timeout.Done() bastionConn.Close() }() - conn, err := bastionConn.Dial("tcp", fmt.Sprintf("%s:22", machineIP)) + conn, err := bastionConn.Dial("tcp", fmt.Sprintf("%s:22", p.machineIP)) if err != nil { - return fmt.Errorf("couldn't dial from bastion host %s to machine %s: %s", bastionIP, machineIP, err) + return fmt.Errorf("couldn't dial from bastion host %s to machine %s: %s", p.bastionIP, p.machineIP, err) } defer conn.Close() cfg = &ssh.ClientConfig{ - User: "ubuntu", + User: p.machineUser, Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), + ssh.PublicKeys(p.signer), }, HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }, Timeout: 60 * time.Second, } cfg.SetDefaults() - Debugf(debug, "dialing from local to machine %s (via tunnel)", machineIP) - clientConn, channels, reqs, err := ssh.NewClientConn(conn, machineIP, cfg) + Debugf(p.debug, "dialing from local to machine %s (via tunnel)", p.machineIP) + clientConn, channels, reqs, err := ssh.NewClientConn(conn, p.machineIP, cfg) if err != nil { - return fmt.Errorf("couldn't dial from local to machine %s: %s", machineIP, err) + return fmt.Errorf("couldn't dial from local to machine %s: %s", p.machineIP, err) } defer clientConn.Close() sshClient := ssh.NewClient(clientConn, channels, reqs) - Debugf(debug, "executing cmd %q on machine %s", cmd.cmd, machineIP) + Debugf(p.debug, "executing cmd %q on machine %s", p.cmd.cmd, p.machineIP) session, err := sshClient.NewSession() if err != nil { - return fmt.Errorf("couldn't open session from local to machine %s to execute cmd %q: %s", cmd.cmd, machineIP, err) + return fmt.Errorf("couldn't open session from local to machine %s to execute cmd %q: %s", p.cmd.cmd, p.machineIP, err) } defer session.Close() - logFile := path.Join(logDir, cmd.title+".log") + logFile := path.Join(p.logDir, p.cmd.title+".log") var stdoutBuf bytes.Buffer var stderrBuf bytes.Buffer session.Stdout = &stdoutBuf session.Stderr = &stderrBuf - if err := session.Run("sudo " + cmd.cmd + "\n"); err != nil { - return fmt.Errorf("unable to send command %q: %s", "sudo "+cmd.cmd, err) + if err := session.Run("sudo " + p.cmd.cmd + "\n"); err != nil { + return fmt.Errorf("unable to send command %q: %s", "sudo "+p.cmd.cmd, err) } result := strings.TrimSuffix(stdoutBuf.String(), "\n") + "\n" + strings.TrimSuffix(stderrBuf.String(), "\n") if err := os.WriteFile(logFile, []byte(result), os.ModePerm); err != nil { return fmt.Errorf("error writing log file: %s", err) } - Debugf(debug, "finished executing cmd %q on machine %s", cmd.cmd, machineIP) + Debugf(p.debug, "finished executing cmd %q on machine %s", p.cmd.cmd, p.machineIP) return nil } diff --git a/test/e2e/shared/exec_test.go b/test/e2e/shared/exec_test.go index 3fcf3a9553..c76ddf5c5d 100644 --- a/test/e2e/shared/exec_test.go +++ b/test/e2e/shared/exec_test.go @@ -88,7 +88,7 @@ func Test_commandsForMachine(t *testing.T) { if err != nil { panic(err) } - executeCommands(context.Background(), filepath.Join(workingDir, "..", "..", "..", "_artifacts"), true, "/tmp/", tt.args.machineIP, tt.args.bastionIP, tt.args.commands) + executeCommands(context.Background(), filepath.Join(workingDir, "..", "..", "..", "_artifacts"), true, "/tmp/", tt.args.machineIP, tt.args.bastionIP, "ubuntu", tt.args.commands) }) } } diff --git a/test/e2e/shared/suite.go b/test/e2e/shared/suite.go index a8064e1364..96eece6fbe 100644 --- a/test/e2e/shared/suite.go +++ b/test/e2e/shared/suite.go @@ -64,6 +64,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte { Expect(e2eCtx.E2EConfig.GetVariable(OpenStackCloudYAMLFile)).To(BeAnExistingFile(), "Invalid test suite argument. Value of environment variable OPENSTACK_CLOUD_YAML_FILE should be an existing file: %s", e2eCtx.E2EConfig.GetVariable(OpenStackCloudYAMLFile)) Logf("Loading the clouds.yaml from %q", e2eCtx.E2EConfig.GetVariable(OpenStackCloudYAMLFile)) + templates := []clusterctl.Files{} + // TODO(sbuerin): we always need ci artifacts, because we don't have images for every Kubernetes version err := filepath.WalkDir(path.Join(e2eCtx.Settings.DataFolder, "infrastructure-openstack"), func(f string, d fs.DirEntry, _ error) error { filename := filepath.Base(f) @@ -102,16 +104,45 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte { TargetName: targetName, } - for i, prov := range e2eCtx.E2EConfig.Providers { - if prov.Name != "openstack" { - continue - } - e2eCtx.E2EConfig.Providers[i].Files = append(e2eCtx.E2EConfig.Providers[i].Files, clusterctlCITemplate) + templates = append(templates, clusterctlCITemplate) + + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + // Walk into the "infrastructure-openstack-no-artifact" directory to simply copy the templates to the "_artifacts/templates" folder. + err = filepath.WalkDir(path.Join(e2eCtx.Settings.DataFolder, "infrastructure-openstack-no-artifact"), func(f string, d fs.DirEntry, _ error) error { + filename := filepath.Base(f) + if d.IsDir() || !strings.HasPrefix(filename, "cluster-template") { + return nil + } + + t, err := os.ReadFile(f) + Expect(err).NotTo(HaveOccurred()) + + targetTemplate := path.Join(e2eCtx.Settings.ArtifactFolder, "templates", filename) + + err = os.WriteFile(targetTemplate, t, 0o600) + Expect(err).NotTo(HaveOccurred()) + + clusterctlTemplate := clusterctl.Files{ + SourcePath: targetTemplate, + TargetName: filename, } + + templates = append(templates, clusterctlTemplate) + return nil }) Expect(err).NotTo(HaveOccurred()) + for i, prov := range e2eCtx.E2EConfig.Providers { + if prov.Name != "openstack" { + continue + } + e2eCtx.E2EConfig.Providers[i].Files = append(e2eCtx.E2EConfig.Providers[i].Files, templates...) + } + ensureSSHKeyPair(e2eCtx) Logf("Creating a clusterctl local repository into %q", e2eCtx.Settings.ArtifactFolder) diff --git a/test/e2e/suites/e2e/e2e_test.go b/test/e2e/suites/e2e/e2e_test.go index 8bcd7f9a98..23028f6c96 100644 --- a/test/e2e/suites/e2e/e2e_test.go +++ b/test/e2e/suites/e2e/e2e_test.go @@ -184,6 +184,79 @@ var _ = Describe("e2e tests [PR-Blocking]", func() { }) }) + Describe("Workload cluster (flatcar)", func() { + It("should be creatable and deletable", func() { + // Flatcar default user is "core" + shared.SetEnvVar(shared.SSHUserMachine, "core", false) + + shared.Logf("Creating a cluster") + clusterName := fmt.Sprintf("cluster-%s", namespace.Name) + configCluster := defaultConfigCluster(clusterName, namespace.Name) + configCluster.ControlPlaneMachineCount = pointer.Int64Ptr(3) + configCluster.WorkerMachineCount = pointer.Int64Ptr(1) + configCluster.Flavor = shared.FlavorFlatcar + md := createCluster(ctx, configCluster) + + workerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + MachineDeployment: *md[0], + }) + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + }) + Expect(workerMachines).To(HaveLen(1)) + Expect(controlPlaneMachines).To(HaveLen(3)) + }) + }) + + Describe("Workload cluster (external cloud provider with Flatcar)", func() { + It("should be creatable and deletable", func() { + // Flatcar default user is "core" + shared.SetEnvVar(shared.SSHUserMachine, "core", false) + + shared.Logf("Creating a cluster") + clusterName := fmt.Sprintf("cluster-%s", namespace.Name) + configCluster := defaultConfigCluster(clusterName, namespace.Name) + configCluster.ControlPlaneMachineCount = pointer.Int64Ptr(1) + configCluster.WorkerMachineCount = pointer.Int64Ptr(1) + configCluster.Flavor = shared.FlavorExternalCloudProviderFlatcar + md := createCluster(ctx, configCluster) + + workerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + MachineDeployment: *md[0], + }) + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + }) + Expect(workerMachines).To(HaveLen(1)) + Expect(controlPlaneMachines).To(HaveLen(1)) + + shared.Logf("Waiting for worker nodes to be in Running phase") + statusChecks := []framework.MachineStatusCheck{framework.MachinePhaseCheck(string(clusterv1.MachinePhaseRunning))} + machineStatusInput := framework.WaitForMachineStatusCheckInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Machine: &workerMachines[0], + StatusChecks: statusChecks, + } + framework.WaitForMachineStatusCheck(ctx, machineStatusInput, e2eCtx.E2EConfig.GetIntervals(specName, "wait-machine-status")...) + + workloadCluster := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName) + + waitForDaemonSetRunning(ctx, workloadCluster.GetClient(), "kube-system", "openstack-cloud-controller-manager") + + waitForNodesReadyWithoutCCMTaint(ctx, workloadCluster.GetClient(), 2) + }) + }) + Describe("Workload cluster (without lb)", func() { It("Should create port(s) with custom options", func() { shared.Logf("Creating a cluster")