From a8167881b0a9e92f412e23ad5052e3e7231e8f76 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 16 Oct 2024 16:35:28 +0100 Subject: [PATCH 1/6] docs: Add note on pod security This is necessary nowadays. Signed-off-by: Stephen Finucane --- docs/content/contribute/develop_in_cluster.md | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/content/contribute/develop_in_cluster.md b/docs/content/contribute/develop_in_cluster.md index 493963dcec..d4cb0830c1 100644 --- a/docs/content/contribute/develop_in_cluster.md +++ b/docs/content/contribute/develop_in_cluster.md @@ -13,7 +13,7 @@ very tedious and slow, the HyperShift project includes a few tools and technique to help make the feedback loop as fast as possible. This guide makes use of the [ko](https://github.com/google/ko) tool to rapidly -build lightweight images which are then published directly into an OCP cluster's +build lightweight images which are then published directly into an OCP cluster's internal registry. This approach has the following properties which can speed up development: @@ -22,7 +22,7 @@ development: - Resulting images are almost as small as the Go binary being published. - Images are published directly into OCP's internal image registry, so images are immediately available on or near the machines that will be pulling them. - + ## Prerequisites - An OCP 4.9+ cluster @@ -93,7 +93,7 @@ make it easy to incorporate the script into pipelines. Pods in the cluster cannot pull the image using the public repo name because the host's certificate is likely self-signed, which would require additional configuration in the cluster to enable pods to pull it. - + Pods must reference the _internal repo pullspec_ as printed to stdout on line 10: `image-registry.openshift-image-registry.svc:5000/hypershift/hypershift-operator-cd22...`. @@ -158,9 +158,18 @@ scaled to 0, enabling developers to replace the components with their own proces (inside or outside the cluster) while preserving the `Deployment` resources to use as templates for the replacement process environments. -For example, the following `HostedCluster` resource will result in a control -plane with the `control-plane-operator` and `ignition-server` deployments -scaled to 0: +For example, to scale the `control-plane-operator` and `ignition-server` deployments +to 0: + +```shell +oc annotate -n clusters HostedCluster test-cluster hypershift.openshift.io/debug-deployments=control-plane-operator,ignition-server +``` + +!!! note + + Update the name of the HostedCluster to match your cluster. + +This will result in a `HostedCluster` like so: ```yaml linenums="1" hl_lines="5" apiVersion: hypershift.openshift.io/v1alpha1 @@ -179,10 +188,17 @@ spec: To scale back up a given component's original deployment simply remove the component's deployment name from the list. +The `hypershift.openshift.io/pod-security-admission-label-override` annotation +may also need to be set in order to run debug pods locally. + +```shell +oc annotate -n clusters HostedCluster test-cluster hypershift.openshift.io/pod-security-admission-label-override=baseline +``` + ## Launch a custom `control-plane-operator` image interactively To iterate on the `control-plane-operator` binary in-cluster interactively, first -[configure the HostedCluster](#configure-a-hostedcluster-for-iterative-control-plane-development) +[configure the HostedCluster](#configure-a-hostedcluster-for-iterative-control-plane-development) to scale down the `control-plane-operator` deployment. Now, you can build and publish the `control-plane-operator` image and run it interactively @@ -202,7 +218,7 @@ press `ctrl-c` to terminate and delete the pod. The default arguments to `control-plane-operator run` should be sufficient to get started. -## Launch a custom ignition server interactively +## Launch a custom `ignition-server` interactively To iterate on the ignition server in-cluster interactively, first [configure the HostedCluster](#configure-a-hostedcluster-for-iterative-control-plane-development) From 9761a635b584525e04ca0c5657e7fb206e0a1018 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 15 Oct 2024 18:40:35 +0100 Subject: [PATCH 2/6] control-plane-operator: Generate assets for OpenStack Cinder CSI Signed-off-by: Stephen Finucane --- .../controllers/resources/resources.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index b78c69988a..036f87a5cf 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -2427,8 +2427,16 @@ func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedCo errs = append(errs, fmt.Errorf("failed to reconcile Storage : %w", err)) } - if hcp.Spec.Platform.Type == hyperv1.AWSPlatform { - driver := manifests.ClusterCSIDriver(operatorv1.AWSEBSCSIDriver) + var driverNames []operatorv1.CSIDriverName + switch hcp.Spec.Platform.Type { + case hyperv1.AWSPlatform: + driverNames = []operatorv1.CSIDriverName{operatorv1.AWSEBSCSIDriver} + case hyperv1.OpenStackPlatform: + // TODO(stephenfin): Add Manila here once it supports Hypershift + driverNames = []operatorv1.CSIDriverName{operatorv1.CinderCSIDriver} + } + for _, driverName := range driverNames { + driver := manifests.ClusterCSIDriver(driverName) if _, err := r.CreateOrUpdate(ctx, r.client, driver, func() error { storage.ReconcileClusterCSIDriver(driver) return nil From 6a5b6ae2e44468746f63cb2ec050664eb3198f81 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 16 Oct 2024 13:28:49 +0100 Subject: [PATCH 3/6] Add missing logs These are present for other reconciliation loops and are very helpful during development. Signed-off-by: Stephen Finucane --- .../controllers/resources/resources.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 036f87a5cf..498f8abd91 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -2409,6 +2409,7 @@ func (r *reconciler) isClusterVersionUpdated(ctx context.Context, version string } func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedControlPlane) []error { + log := ctrl.LoggerFrom(ctx) var errs []error snapshotController := manifests.CSISnapshotController() @@ -2417,16 +2418,22 @@ func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedCo return nil }); err != nil { errs = append(errs, fmt.Errorf("failed to reconcile CSISnapshotController : %w", err)) + } else { + log.Info("reconciled CSISnapshotController") } + storageCR := manifests.Storage() if _, err := r.CreateOrUpdate(ctx, r.client, storageCR, func() error { storage.ReconcileStorage(storageCR) return nil }); err != nil { errs = append(errs, fmt.Errorf("failed to reconcile Storage : %w", err)) + } else { + log.Info("reconciled Storage") } + var driverNames []operatorv1.CSIDriverName switch hcp.Spec.Platform.Type { case hyperv1.AWSPlatform: @@ -2442,6 +2449,8 @@ func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedCo return nil }); err != nil { errs = append(errs, fmt.Errorf("failed to reconcile ClusterCSIDriver %s: %w", driver.Name, err)) + } else { + log.Info("reconciled ClusterCSIDriver %s", "name", driver.Name) } } return errs From bfef53aad83834d0b5015b1da09ca61c6c9593b9 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 23 Oct 2024 15:10:17 -0400 Subject: [PATCH 4/6] openstack: reconcile CSI driver secret for OSP credentials --- .../internal/platform/openstack/openstack.go | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go b/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go index 8536a986e1..977b2beea3 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go @@ -275,24 +275,38 @@ func (a OpenStack) ReconcileCredentials(ctx context.Context, c client.Client, cr } // Sync CNCC secret + if err := a.reconcileOpenStackCredentialsSecret(ctx, c, createOrUpdate, hcluster, controlPlaneNamespace, "cloud-network-config-controller-creds"); err != nil { + return err + } + // Sync CSI driver secret + if err := a.reconcileOpenStackCredentialsSecret(ctx, c, createOrUpdate, hcluster, controlPlaneNamespace, "openstack-cloud-credentials"); err != nil { + return err + } + + return nil +} + +// reconcileOpenStackCredentialsSecret is a wrapper used ot reconcile the OpenStack cloud config secret. +func (a OpenStack) reconcileOpenStackCredentialsSecret(ctx context.Context, c client.Client, createOrUpdate upsert.CreateOrUpdateFN, hcluster *hyperv1.HostedCluster, controlPlaneNamespace, name string) error { credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcluster.Namespace, Name: hcluster.Spec.Platform.OpenStack.IdentityRef.Name}} if err := c.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { return fmt.Errorf("failed to get OpenStack credentials secret: %w", err) } + caCertData := openstack.GetCACertFromCredentialsSecret(credentialsSecret) - cloudNetworkConfigCreds := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cloud-network-config-controller-creds"}, + credsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: name}, Data: map[string][]byte{}, } - cloudNetworkConfigCreds.Data[openstack.CloudsSecretKey] = credentialsSecret.Data[openstack.CloudsSecretKey] + credsSecret.Data[openstack.CloudsSecretKey] = credentialsSecret.Data[openstack.CloudsSecretKey] if caCertData != nil { - cloudNetworkConfigCreds.Data[openstack.CABundleKey] = caCertData + credsSecret.Data[openstack.CABundleKey] = caCertData } - if _, err := createOrUpdate(ctx, c, cloudNetworkConfigCreds, func() error { - return openstack.ReconcileCloudConfigSecret(hcluster.Spec.Platform.OpenStack.ExternalNetwork.ID, cloudNetworkConfigCreds, hcluster.Spec.Platform.OpenStack.IdentityRef.CloudName, credentialsSecret, caCertData) + if _, err := createOrUpdate(ctx, c, credsSecret, func() error { + return openstack.ReconcileCloudConfigSecret(hcluster.Spec.Platform.OpenStack.ExternalNetwork.ID, credsSecret, hcluster.Spec.Platform.OpenStack.IdentityRef.CloudName, credentialsSecret, caCertData) }); err != nil { - return fmt.Errorf("failed to reconcile OpenStack cloud config: %w", err) + return fmt.Errorf("failed to reconcile OpenStack cloud config for %s: %w", name, err) } return nil From de62b8ebe4b60d258236fa172a15cc1c20a26d3d Mon Sep 17 00:00:00 2001 From: Maysa Macedo Date: Fri, 25 Oct 2024 12:37:13 -0300 Subject: [PATCH 5/6] Add Manila CSI --- .../controllers/resources/resources.go | 5 +---- .../hostedcluster/internal/platform/openstack/openstack.go | 7 ++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 498f8abd91..40ad2c29a2 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -2422,7 +2422,6 @@ func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedCo log.Info("reconciled CSISnapshotController") } - storageCR := manifests.Storage() if _, err := r.CreateOrUpdate(ctx, r.client, storageCR, func() error { storage.ReconcileStorage(storageCR) @@ -2433,14 +2432,12 @@ func (r *reconciler) reconcileStorage(ctx context.Context, hcp *hyperv1.HostedCo log.Info("reconciled Storage") } - var driverNames []operatorv1.CSIDriverName switch hcp.Spec.Platform.Type { case hyperv1.AWSPlatform: driverNames = []operatorv1.CSIDriverName{operatorv1.AWSEBSCSIDriver} case hyperv1.OpenStackPlatform: - // TODO(stephenfin): Add Manila here once it supports Hypershift - driverNames = []operatorv1.CSIDriverName{operatorv1.CinderCSIDriver} + driverNames = []operatorv1.CSIDriverName{operatorv1.CinderCSIDriver, operatorv1.ManilaCSIDriver} } for _, driverName := range driverNames { driver := manifests.ClusterCSIDriver(driverName) diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go b/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go index 977b2beea3..cc66c84dd8 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/openstack/openstack.go @@ -278,11 +278,16 @@ func (a OpenStack) ReconcileCredentials(ctx context.Context, c client.Client, cr if err := a.reconcileOpenStackCredentialsSecret(ctx, c, createOrUpdate, hcluster, controlPlaneNamespace, "cloud-network-config-controller-creds"); err != nil { return err } - // Sync CSI driver secret + // Sync Cinder CSI driver secret if err := a.reconcileOpenStackCredentialsSecret(ctx, c, createOrUpdate, hcluster, controlPlaneNamespace, "openstack-cloud-credentials"); err != nil { return err } + // Sync Manila CSI driver secret + if err := a.reconcileOpenStackCredentialsSecret(ctx, c, createOrUpdate, hcluster, controlPlaneNamespace, "manila-cloud-credentials"); err != nil { + return err + } + return nil } From 3e15d2dad8ff0447bda30c1d18ac8ad6e257463e Mon Sep 17 00:00:00 2001 From: Maysa Macedo Date: Wed, 30 Oct 2024 16:08:43 -0300 Subject: [PATCH 6/6] Set control plane image value --- .../storage/assets/10_deployment-hypershift.yaml | 2 ++ .../controllers/hostedcontrolplane/storage/envreplace.go | 1 + 2 files changed, 3 insertions(+) diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/assets/10_deployment-hypershift.yaml b/control-plane-operator/controllers/hostedcontrolplane/storage/assets/10_deployment-hypershift.yaml index 698d8c1b0a..350c38ebfb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/assets/10_deployment-hypershift.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/assets/10_deployment-hypershift.yaml @@ -111,6 +111,8 @@ spec: value: quay.io/openshift/origin-azure-file-csi-driver-operator:latest - name: TOOLS_IMAGE value: quay.io/openshift/origin-tools:latest + - name: MANILA_DRIVER_CONTROL_PLANE_IMAGE + value: quay.io/openshift/origin-csi-driver-manila-operator:latest image: quay.io/openshift/origin-cluster-storage-operator:latest imagePullPolicy: IfNotPresent name: cluster-storage-operator diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/envreplace.go b/control-plane-operator/controllers/hostedcontrolplane/storage/envreplace.go index 5d13f4a516..9756104878 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/envreplace.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/envreplace.go @@ -51,6 +51,7 @@ var ( "AWS_EBS_DRIVER_CONTROL_PLANE_IMAGE": "aws-ebs-csi-driver", "AZURE_DISK_DRIVER_CONTROL_PLANE_IMAGE": "azure-disk-csi-driver", "AZURE_FILE_DRIVER_CONTROL_PLANE_IMAGE": "azure-file-csi-driver", + "MANILA_DRIVER_CONTROL_PLANE_IMAGE": "csi-driver-manila", "LIVENESS_PROBE_CONTROL_PLANE_IMAGE": "csi-livenessprobe", "TOOLS_IMAGE": "tools", }