Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Map PV secrets in parameters when restoring PVs #1602

Merged
merged 2 commits into from
Nov 5, 2024
Merged

Map PV secrets in parameters when restoring PVs #1602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f050062
Map PV secrets in parameters when restoring PVs
rakeshgm Oct 23, 2024
e1e279f
fix vrg_tests
rakeshgm Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 101 additions & 21 deletions internal/controller/vrg_volrep.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,21 @@ const (
defaultVRCAnnotationKey = "replication.storage.openshift.io/is-default-class"
)

//nolint:gosec
const (
// secretRef keys
controllerPublishSecretName = "csi.storage.k8s.io/controller-publish-secret-name"
Copy link
Member

@Madhu-1 Madhu-1 Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about the provisioner secrets that are present in pv annotation? Want we update that as well?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like am late 😕

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Provisioner secrets are in PV annotations? I assume these are needed for Delete operations?

If so @rakeshgm add a new PR to update that as well.

Copy link
Member

@Madhu-1 Madhu-1 Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

apiVersion: v1
items:
- apiVersion: v1
  kind: PersistentVolume
  metadata:
    annotations:
      pv.kubernetes.io/provisioned-by: rook-ceph.rbd.csi.ceph.com
      volume.kubernetes.io/provisioner-deletion-secret-name: rook-csi-rbd-provisioner
      volume.kubernetes.io/provisioner-deletion-secret-namespace: rook-ceph
    creationTimestamp: "2024-11-05T09:51:09Z"
    finalizers:

Yes in the pv annotation for delete operation.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a task in #1616

controllerPublishSecretNamespace = "csi.storage.k8s.io/controller-publish-secret-namespace"
nodeStageSecretName = "csi.storage.k8s.io/node-stage-secret-name"
nodeStageSecretNamespace = "csi.storage.k8s.io/node-stage-secret-namespace"
nodePublishSecretName = "csi.storage.k8s.io/node-publish-secret-name"
nodePublishSecretNamespace = "csi.storage.k8s.io/node-publish-secret-namespace"
controllerExpandSecretName = "csi.storage.k8s.io/controller-expand-secret-name"
controllerExpandSecretNamespace = "csi.storage.k8s.io/controller-expand-secret-namespace"
nodeExpandSecretName = "csi.storage.k8s.io/node-expand-secret-name"
nodeExpandSecretNamespace = "csi.storage.k8s.io/node-expand-secret-namespace"
)

func logWithPvcName(log logr.Logger, pvc *corev1.PersistentVolumeClaim) logr.Logger {
return log.WithValues("pvc", types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}.String())
}
Expand Down Expand Up @@ -1332,6 +1347,24 @@ func (v *VRGInstance) filterDefaultVRC(
defaultVRCAnnotationKey)
}

func (v *VRGInstance) getStorageClassFromSCName(scName *string) (*storagev1.StorageClass, error) {
if storageClass, ok := v.storageClassCache[*scName]; ok {
return storageClass, nil
}

storageClass := &storagev1.StorageClass{}
if err := v.reconciler.Get(v.ctx, types.NamespacedName{Name: *scName}, storageClass); err != nil {
v.log.Info(fmt.Sprintf("Failed to get the storageclass %s", *scName))

return nil, fmt.Errorf("failed to get the storageclass with name %s (%w)",
*scName, err)
}

v.storageClassCache[*scName] = storageClass

return storageClass, nil
}

// getStorageClass inspects the PVCs being protected by this VRG instance for the passed in namespacedName, and
// returns its corresponding StorageClass resource from an instance cache if available, or fetches it from the API
// server and stores it in an instance cache before returning the StorageClass
Expand Down Expand Up @@ -1363,21 +1396,7 @@ func (v *VRGInstance) getStorageClass(namespacedName types.NamespacedName) (*sto
return nil, fmt.Errorf("missing StorageClass name for pvc (%s)", namespacedName)
}

if storageClass, ok := v.storageClassCache[*scName]; ok {
return storageClass, nil
}

storageClass := &storagev1.StorageClass{}
if err := v.reconciler.Get(v.ctx, types.NamespacedName{Name: *scName}, storageClass); err != nil {
v.log.Info(fmt.Sprintf("Failed to get the storageclass %s", *scName))

return nil, fmt.Errorf("failed to get the storageclass with name %s (%w)",
*scName, err)
}

v.storageClassCache[*scName] = storageClass

return storageClass, nil
return v.getStorageClassFromSCName(scName)
}

// checkVRStatus checks if the VolumeReplication resource has the desired status for the
Expand Down Expand Up @@ -2025,7 +2044,7 @@ func (v *VRGInstance) restorePVsFromObjectStore(objectStore ObjectStorer, s3Prof
return 0, fmt.Errorf("%s: %w", errMsg, err)
}

return restoreClusterDataObjects(v, pvList, "PV", cleanupPVForRestore, v.validateExistingPV)
return restoreClusterDataObjects(v, pvList, "PV", v.cleanupPVForRestore, v.validateExistingPV)
}

func (v *VRGInstance) restorePVCsFromObjectStore(objectStore ObjectStorer, s3ProfileName string) (int, error) {
Expand Down Expand Up @@ -2092,7 +2111,7 @@ func restoreClusterDataObjects[
](
v *VRGInstance,
objList []ObjectType, objType string,
cleanupForRestore func(*ObjectType),
cleanupForRestore func(*ObjectType) error,
validateExistingObject func(*ObjectType) error,
) (int, error) {
numRestored := 0
Expand All @@ -2102,7 +2121,13 @@ func restoreClusterDataObjects[
objectCopy := &*object
obj := ClientObject(objectCopy)

cleanupForRestore(objectCopy)
err := cleanupForRestore(objectCopy)
if err != nil {
v.log.Info("failed to cleanup during restore", "error", err.Error())

return numRestored, err
}

addRestoreAnnotation(obj)

if err := v.reconciler.Create(v.ctx, obj); err != nil {
Expand Down Expand Up @@ -2142,7 +2167,11 @@ func (v *VRGInstance) updateExistingPVForSync(pv *corev1.PersistentVolume) error
// failover/relocate process. Hence, the restore may not be
// required and the annotation for restore can be missing for
// the sync mode.
cleanupPVForRestore(pv)
err := v.cleanupPVForRestore(pv)
if err != nil {
return err
}

addRestoreAnnotation(pv)

if err := v.reconciler.Update(v.ctx, pv); err != nil {
Expand Down Expand Up @@ -2300,22 +2329,73 @@ func addRestoreAnnotation(obj client.Object) {
obj.GetAnnotations()[RestoreAnnotation] = RestoredByRamen
}

func secretsFromSC(params map[string]string,
secretName, secretNamespace string,
) (*corev1.SecretReference, bool) {
secretRef := corev1.SecretReference{
Name: params[secretName],
Namespace: params[secretNamespace],
}

exists := secretRef != (corev1.SecretReference{})

return &secretRef, exists
}

func (v *VRGInstance) processPVSecrets(pv *corev1.PersistentVolume) error {
sc, err := v.getStorageClassFromSCName(&pv.Spec.StorageClassName)
if err != nil {
return err
}

secFromSC, exists := secretsFromSC(sc.Parameters, nodeStageSecretName, nodeStageSecretNamespace)
if exists {
pv.Spec.CSI.NodeStageSecretRef = secFromSC
}

secFromSC, exists = secretsFromSC(sc.Parameters, nodePublishSecretName, nodePublishSecretNamespace)
if exists {
pv.Spec.CSI.NodePublishSecretRef = secFromSC
}

secFromSC, exists = secretsFromSC(sc.Parameters, nodeExpandSecretName, nodeExpandSecretNamespace)
if exists {
pv.Spec.CSI.NodeExpandSecretRef = secFromSC
}

secFromSC, exists = secretsFromSC(sc.Parameters, controllerExpandSecretName, controllerExpandSecretNamespace)
if exists {
pv.Spec.CSI.ControllerExpandSecretRef = secFromSC
}

secFromSC, exists = secretsFromSC(sc.Parameters, controllerPublishSecretName, controllerPublishSecretNamespace)
if exists {
pv.Spec.CSI.ControllerExpandSecretRef = secFromSC
}

return nil
}

// cleanupForRestore cleans up required PV or PVC fields, to ensure restore succeeds
// to a new cluster, and rebinding the PVC to an existing PV with the same claimRef
func cleanupPVForRestore(pv *corev1.PersistentVolume) {
func (v *VRGInstance) cleanupPVForRestore(pv *corev1.PersistentVolume) error {
pv.ResourceVersion = ""
if pv.Spec.ClaimRef != nil {
pv.Spec.ClaimRef.UID = ""
pv.Spec.ClaimRef.ResourceVersion = ""
pv.Spec.ClaimRef.APIVersion = ""
}

return v.processPVSecrets(pv)
}

func cleanupPVCForRestore(pvc *corev1.PersistentVolumeClaim) {
func cleanupPVCForRestore(pvc *corev1.PersistentVolumeClaim) error {
pvc.ObjectMeta.Annotations = PruneAnnotations(pvc.GetAnnotations())
pvc.ObjectMeta.Finalizers = []string{}
pvc.ObjectMeta.ResourceVersion = ""
pvc.ObjectMeta.OwnerReferences = nil

return nil
}

// Follow this logic to update VRG (and also ProtectedPVC) conditions for VolRep
Expand Down
Loading
Loading