Notifications - PR notes, take1

Signed-off-by: Amit Prinz Setter <[email protected]>

deploy/crds/noobaa.io_noobaas.yaml

@@ -1028,6 +1028,40 @@ spec:
                       - guaranteed - much more reliable but need to provide a storage class that supports RWX PVs
                     type: string
                 type: object
+              bucketNotifications:
+                description: BucketNotifications (optional) controls bucket notification
+                  options
+                properties:
+                  connections:
+                    description: |-
+                      Connections - A list of secrets' names that are used by the notifications configrations
+                      (in the TopicArn field).
+                    items:
+                      description: |-
+                        SecretReference represents a Secret Reference. It has enough information to retrieve secret
+                        in any namespace
+                      properties:
+                        name:
+                          description: name is unique within a namespace to reference
+                            a secret resource.
+                          type: string
+                        namespace:
+                          description: namespace defines the space within which the
+                            secret name must be unique.
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
+                  enabled:
+                    description: Enabled - whether bucket notifications is enabled
+                    type: boolean
+                  pvc:
+                    description: |-
+                      PVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used
+                      for holding pending notifications files.
+                      For ODF - If not provided, the default CepthFS storage class will be used to create the PVC.
+                    type: string
+                type: object
               cleanupPolicy:
                 description: CleanupPolicy (optional) Indicates user's policy for
                   deletion
@@ -1706,11 +1740,6 @@ spec:
                   update the admin account with new BackingStore/NamespaceStore in order to delete the DefaultBackingStore/DefaultNamespaceStore
                 nullable: true
                 type: boolean
-              notificationsPVC:
-                description: |-
-                  NotificationsPVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used
-                  for notifications persistent files.
-                type: string
               pvPoolDefaultStorageClass:
                 description: |-
                   PVPoolDefaultStorageClass (optional) overrides the default cluster StorageClass for the pv-pool volumes.

pkg/apis/noobaa/v1alpha1/noobaa_types.go

@@ -228,10 +228,9 @@ type NooBaaSpec struct {
 	// +optional
 	BucketLogging BucketLoggingSpec `json:"bucketLogging,omitempty"`
 
-	// NotificationsPVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used
-	// for notifications persistent files.
+	// BucketNotifications (optional) controls bucket notification options
 	// +optional
-	NotificationsPVC *string `json:"notificationsPVC,omitempty"`
+	BucketNotifications BucketNotificationsSpec `json:"bucketNotifications,omitempty"`
 }
 
 // AutoscalerSpec defines different actoscaling spec such as autoscaler type and prometheus namespace
@@ -263,6 +262,22 @@ type BucketLoggingSpec struct {
 	BucketLoggingPVC *string `json:"bucketLoggingPVC,omitempty"`
 }
 
+//BucketNotificationsSpec controls bucket notification configuration
+type BucketNotificationsSpec struct {
+	// Enabled - whether bucket notifications is enabled
+	Enabled bool `json:"enabled,omitempty"`
+
+	//PVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used
+	//for holding pending notifications files.
+	//For ODF - If not provided, the default CepthFS storage class will be used to create the PVC.
+	// +optional
+	PVC *string `json:"pvc,omitempty"`
+
+	//Connections - A list of secrets' names that are used by the notifications configrations
+	//(in the TopicArn field).
+	Connections []corev1.SecretReference `json:"connections,omitempty"`
+}
+
 // LoadBalancerSourceSubnetSpec defines the subnets that will be allowed to access the NooBaa services
 type LoadBalancerSourceSubnetSpec struct {
 	// S3 is a list of subnets that will be allowed to access the Noobaa S3 service

pkg/bundle/deploy.go

@@ -1415,7 +1415,7 @@ spec:
       status: {}
 `
 
-const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "0f61ab9eecb4ccd392e5e5e7932baa0fa12ba035b56c4d610a8eaaebb98c8188"
+const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "3f88c800238f25e5dd26f3f1bf19028571cc646e3aea0f868bfd2ff600ee3ed1"
 
 const File_deploy_crds_noobaa_io_noobaas_yaml = `---
 apiVersion: apiextensions.k8s.io/v1
@@ -3125,11 +3125,6 @@ spec:
                   update the admin account with new BackingStore/NamespaceStore in order to delete the DefaultBackingStore/DefaultNamespaceStore
                 nullable: true
                 type: boolean
-              notificationsPVC:
-                description: |-
-                  NotificationsPVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used
-                  for notifications persistent files.
-                type: string
               pvPoolDefaultStorageClass:
                 description: |-
                   PVPoolDefaultStorageClass (optional) overrides the default cluster StorageClass for the pv-pool volumes.

pkg/system/phase2_creating.go

@@ -136,6 +136,14 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 			return err
 		}
 	}
+
+	// create notification log pvc if bucket notifications is enabled and pvc was not set explicitly
+	if r.NooBaa.Spec.BucketNotifications.Enabled {
+		if err := r.ReconcileODFNotificationsPVC(); err != nil {
+			return err
+		}
+	}
+
 	util.KubeCreateOptional(util.KubeObject(bundle.File_deploy_scc_core_yaml).(*secv1.SecurityContextConstraints))
 	if err := r.ReconcileObject(r.ServiceMgmt, r.SetDesiredServiceMgmt); err != nil {
 		return err
@@ -451,12 +459,11 @@ func (r *Reconciler) setDesiredCoreEnv(c *corev1.Container) {
 		}
 	}
 
-	if r.NooBaa.Spec.NotificationsPVC != nil {
+	if r.NooBaa.Spec.BucketNotifications.Enabled {//TODO - has pvc
 		envVar := corev1.EnvVar{
 			Name: "NOTIFICATION_LOG_DIR",
 			Value: "/var/logs/notifications",
 		}
-
 		util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar});
 	}
 
@@ -530,7 +537,7 @@ func (r *Reconciler) SetDesiredCoreApp() error {
 				util.MergeVolumeMountList(&c.VolumeMounts, &bucketLogVolumeMounts)
 			}
 
-			if r.NooBaa.Spec.NotificationsPVC != nil {
+			if r.NooBaa.Spec.BucketNotifications.Enabled {
 				notificationVolumeMounts := []corev1.VolumeMount{{
 					Name:      "notif-vol",
 					MountPath: "/var/logs/notifications",
@@ -541,32 +548,25 @@ func (r *Reconciler) SetDesiredCoreApp() error {
 					Name: "notif-vol",
 					VolumeSource: corev1.VolumeSource {
 						PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource {
-							ClaimName: *r.NooBaa.Spec.NotificationsPVC,
+							ClaimName: r.BucketNotificationsPVC.Name,
 						},
 					},
 				}}
 				util.MergeVolumeList(&podSpec.Volumes, &notificationVolumes)
 
-				//find secrets that tell us how to connect to remote notifications servers,
-				//mount them so core can read them
-				notificatoinSecrets := &corev1.SecretList{}
-				noobaaNotifSelector, _ := labels.Parse("app=noobaa,noobaa=notifications")
-				util.KubeList(notificatoinSecrets, &client.ListOptions{Namespace: options.Namespace, LabelSelector: noobaaNotifSelector})
-
-				for _, notificationSecret := range notificatoinSecrets.Items {
-
+				for _, notifSecret := range r.NooBaa.Spec.BucketNotifications.Connections {
 					secretVolumeMounts := []corev1.VolumeMount{{
-						Name:      notificationSecret.Name,
-						MountPath: "/etc/notif_connect/" + notificationSecret.Name,
+						Name:      notifSecret.Name,
+						MountPath: "/etc/notif_connect/" + notifSecret.Name,
 						ReadOnly:  true,
 					}}
 					util.MergeVolumeMountList(&c.VolumeMounts, &secretVolumeMounts)
 
 					secretVolumes := []corev1.Volume{{
-						Name: notificationSecret.Name,
+						Name: notifSecret.Name,
 						VolumeSource: corev1.VolumeSource{
 							Secret: &corev1.SecretVolumeSource{
-								SecretName: notificationSecret.Name,
+								SecretName: notifSecret.Name,
 							},
 						},
 					}}
@@ -1271,6 +1271,58 @@ func (r *Reconciler) prepareODFBucketLoggingPVC() error {
 	return nil
 }
 
+// ReconcileODFNotificationsPVC ensures the bucket notifications PVC is properly configured
+func (r *Reconciler) ReconcileODFNotificationsPVC() error {
+	log := r.Logger.WithField("func", "ReconcileNotificationsPVC")
+
+	// Return if notifications PVC already exist
+	if r.NooBaa.Spec.BucketNotifications.PVC != nil {
+		r.BucketNotificationsPVC.Name = *r.NooBaa.Spec.BucketNotifications.PVC
+		log.Infof("BucketNotifications.pvc %s already exists and supports RWX access mode. Skipping ReconcileODFBucketLoggingPVC.", r.BucketNotificationsPVC.Name)
+		return nil
+	}
+
+	util.KubeCheck(r.BucketNotificationsPVC)
+	if r.BucketNotificationsPVC.UID != "" {
+		log.Infof("BucketNotifications.pvc %s already exists. Skipping creation.", r.BucketNotificationsPVC.Name)
+		return nil
+	}
+
+	if err := r.prepareODFNotificationsPVC(); err != nil {
+		return err
+	}
+	r.Own(r.BucketNotificationsPVC)
+
+	log.Infof("BucketNotifications.pvc %s does not exist. Creating...", r.BucketNotificationsPVC.Name)
+	err := r.Client.Create(r.Ctx, r.BucketNotificationsPVC)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// prepareODFNotificationsPVC configures the bucket notifications pvc
+func (r *Reconciler) prepareODFNotificationsPVC() error {
+	r.BucketNotificationsPVC.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}
+
+	sc := &storagev1.StorageClass{
+		TypeMeta:   metav1.TypeMeta{Kind: "StorageClass"},
+		ObjectMeta: metav1.ObjectMeta{Name: "ocs-storagecluster-cephfs"},
+	}
+
+	// fallback to cephfs storageclass to create bucket logging pvc if running on ODF
+	if util.KubeCheck(sc) {
+		r.Logger.Infof("BucketNotifications.pvc not provided, defaulting to 'cephfs' storage class %s to create bucket notifications pvc", sc.Name)
+		r.BucketNotificationsPVC.Spec.StorageClassName = &sc.Name
+	} else {
+		return util.NewPersistentError("InvalidBucketNotificationsConfiguration",
+			"Bucket Notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify 'BucketNotifications.pvc'.")
+	}
+
+	return nil
+}
+
 // SetDesiredPostgresDBConf fill desired postgres db config map
 func (r *Reconciler) SetDesiredPostgresDBConf() error {
 	dbConfigYaml := util.KubeObject(bundle.File_deploy_internal_configmap_postgres_db_yaml).(*corev1.ConfigMap)

pkg/system/phase4_configuring.go

@@ -415,7 +415,7 @@ func (r *Reconciler) SetDesiredDeploymentEndpoint() error {
 				}
 			}
 
-			if r.NooBaa.Spec.NotificationsPVC != nil {
+			if r.NooBaa.Spec.BucketNotifications.Enabled {
 				envVar := corev1.EnvVar{
 					Name: "NOTIFICATION_LOG_DIR",
 					Value: "/var/logs/notifications",
@@ -542,12 +542,12 @@ func (r *Reconciler) setDesiredEndpointMounts(podSpec *corev1.PodSpec, container
 		util.MergeVolumeMountList(&container.VolumeMounts, &bucketLogVolumeMounts)
 	}
 
-	if r.NooBaa.Spec.NotificationsPVC != nil {
+	if r.NooBaa.Spec.BucketNotifications.Enabled {
 		notificationVolumes := []corev1.Volume{{
 			Name: "notif-vol",
 			VolumeSource: corev1.VolumeSource{
 				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-					ClaimName: *r.NooBaa.Spec.NotificationsPVC,
+					ClaimName: r.BucketNotificationsPVC.Name,
 				},
 			},
 		}}

pkg/system/reconciler.go

@@ -122,6 +122,7 @@ type Reconciler struct {
 	AdapterHPA                *autoscalingv2.HorizontalPodAutoscaler
 	ExternalPgSecret          *corev1.Secret
 	ExternalPgSSLSecret       *corev1.Secret
+	BucketNotificationsPVC    *corev1.PersistentVolumeClaim
 }
 
 // NewReconciler initializes a reconciler to be used for loading or reconciling a noobaa system
@@ -171,6 +172,7 @@ func NewReconciler(
 		DefaultNsfsPvc:            util.KubeObject(bundle.File_deploy_internal_nsfs_pvc_cr_yaml).(*corev1.PersistentVolumeClaim),
 		OBCStorageClass:           util.KubeObject(bundle.File_deploy_obc_storage_class_yaml).(*storagev1.StorageClass),
 		BucketLoggingPVC:          util.KubeObject(bundle.File_deploy_internal_pvc_agent_yaml).(*corev1.PersistentVolumeClaim),
+		BucketNotificationsPVC:    util.KubeObject(bundle.File_deploy_internal_pvc_agent_yaml).(*corev1.PersistentVolumeClaim),
 		PrometheusRule:            util.KubeObject(bundle.File_deploy_internal_prometheus_rules_yaml).(*monitoringv1.PrometheusRule),
 		ServiceMonitorMgmt:        util.KubeObject(bundle.File_deploy_internal_servicemonitor_mgmt_yaml).(*monitoringv1.ServiceMonitor),
 		ServiceMonitorS3:          util.KubeObject(bundle.File_deploy_internal_servicemonitor_s3_yaml).(*monitoringv1.ServiceMonitor),
@@ -230,6 +232,7 @@ func NewReconciler(
 	r.KedaScaled.Namespace = r.Request.Namespace
 	r.AdapterHPA.Namespace = r.Request.Namespace
 	r.BucketLoggingPVC.Namespace = r.Request.Namespace
+	r.BucketNotificationsPVC.Namespace = r.Request.Namespace
 
 	// Set Names
 	r.NooBaa.Name = r.Request.Name
@@ -274,6 +277,7 @@ func NewReconciler(
 	r.KedaScaled.Name = r.Request.Name
 	r.AdapterHPA.Name = r.Request.Name + "-hpav2"
 	r.BucketLoggingPVC.Name = r.Request.Name + "-bucket-logging-pvc"
+	r.BucketNotificationsPVC.Name = r.Request.Name + "-bucket-notifications-pvc"
 
 	// Set the target service for routes.
 	r.RouteMgmt.Spec.To.Name = r.ServiceMgmt.Name