From 7ef2df1b182b5117aa231e85f267e69e1b10b951 Mon Sep 17 00:00:00 2001 From: Dee Kryvenko <109895+dee-kryvenko@users.noreply.github.com> Date: Sun, 11 Dec 2022 22:06:59 -0800 Subject: [PATCH] Replace Releaser association mechanism through PVC with a Storage Class annotation (#15) Closes #13 --- CHANGELOG.md | 36 +++ Dockerfile | 4 + README.md | 32 +-- cmd/reclaimable-pv-releaser/main.go | 7 +- controller.go | 10 + examples/basic/README.md | 18 +- examples/basic/sc.yaml | 2 + .../README.md | 4 + releaser/releaser.go | 213 +++++++++++------- 9 files changed, 218 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51fe8b8..436c0a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2022-12-11 + +### BREAKING CHANGES + +Prior to this change, one of the features of Releaser was to "Automatically associates Releaser with PVs claimed by PVCs that were created by Provisioner with the same `--controller-id`". +From the `README.md` prior to this change: + +> ## PV Releaser Controller +> +> For Releaser to be able to make PVs claimed by Provisioner `Available` after PVC is gone - Releaser and Provisioner must share the same Controller ID. +> +> ### Associate +> +> Once `Released` - PVs doesn't have any indication that they were once associated with a PVC that had association with this Controller ID. To establish this relation - we must catch it while PVC still exists and mark it with our label. If Releaser was down the whole time PVC existed - PV could never be associated making it now orphaned and it will stay as `Released` - Releaser can't know it have to make it `Available`. +> +> Releaser listens for PV creations/updates. +> The following conditions must be met for a PV to be associated with a Releaser: +> +> - PV doesn't already have `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` association. +> - `spec.claimRef` must refer to a PVC that either has `metadata.labels."dynamic-pvc-provisioner.kubernetes.io/managed-by"` or `reclaimable-pv-releaser.kubernetes.io/managed-by` set to this Controller ID. If both labels are set - both should point to this Controller ID. +> - `--disable-automatic-association` must be `false`. +> +> To establish association Releaser will set itself to `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` on this PV. + +As disclaimed - that approach was error prone. It was fine most of the time, but if Releaser was down for any noticeable duration of time - it was resulting in PVs piling up in `Released` state, and as the PVC was long gone by then - PVs would remain in that state forever, until manually cleared up. + +This mechanism of association through PVC was removed in this release and replaced with a simple Storage Class annotation. In order for Releaser to turn `Released` PV as `Available` - its Storage Class must be annotated with `metadata.annotations."reclaimable-pv-releaser.kubernetes.io/controller-id"` pointing at the `-controller-id` of this Releaser. It can now retro-actively release PVs on startup that it never received events about. As a side effect - `-controller-id` of Provisioner and Releaser doesn't have to match anymore. This unfortunately requires that you use dedicated Storage Class for PVs that must be reclaimable, but that is a fair price to pay if the alternative is unreliable and error prone and might result in expensive storage bills. + +You must use Helm charts version `v0.1.0` or above as RBAC is changed in this release to allow read-only access to Storage Classes. + +### Changed + +- Old PV association via PVC mechanism was removed +- `-disable-automatic-association` option on Releaser was removed +- PVs will only be released now if their Storage Class annotated with `metadata.annotations."reclaimable-pv-releaser.kubernetes.io/controller-id"` pointing at the `-controller-id` of this Releaser + ## [0.1.1] - 2022-12-11 ### Fixed diff --git a/Dockerfile b/Dockerfile index a0590d4..b7e9962 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,10 @@ FROM golang:1.18 AS build ARG RELEASE_STRING=dev ENV IMPORT_PATH="github.com/plumber-cd/kubernetes-dynamic-reclaimable-pvc-controllers" WORKDIR /go/delivery + +COPY go.mod go.sum ./ +RUN go mod download + COPY . . RUN mkdir bin && go build \ -ldflags "-X ${IMPORT_PATH}.Version=${RELEASE_STRING}" \ diff --git a/README.md b/README.md index cb534c3..e6523bb 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,15 @@ Dynamic PVC provisioner for pods requesting it via annotations. Automatic PV rel - Dynamically create PVC for Pods requesting it via the annotations. - Pod is automatically set as `ownerReferences` to the PVC - guaranteeing its deletions upon Pod deletion. - PV Releaser - - Automatically associates Releaser with PVs claimed by PVCs that were created by Provisioner with the same `--controller-id`. - - Deletes `claimRef` from PVs associated with Releaser to move their status from `Released` to `Available` **without deleting any data**. + - Keeps track of Storage Classes marked by annotation pointing to the same `--controller-id`. + - Deletes `claimRef` from PVs associated with Releaser (via Storage Class annotation) to move their status from `Released` to `Available` **without deleting any data**. - Provisioner and Releaser are two separate controllers under one roof, and they can be deployed separately. - You can use Provisioner alone for something like Jenkins Kubernetes plugin that doesn't allow PVC creation on its own and automate PVC provisioning from the pod requests. Provisioner on its own will not make PVs automatically reclaimable. - - You can use Releaser alone - provided you associate either your PVCs or PVs with it on your own. That will set PVCs able to automatically reclaim associated PVs with whatever data left in it from previous consumer. -- To make Releaser and Deployer work together - they need to have the same `--controller-id`. + - You can use Releaser alone. That will enable PVCs to automatically reclaim PVs with whatever data left in it from previous consumer. ## Disclaimers -**Provisioner Controller ignores RBAC. If the user creating the Pod didn't had permissions to create PVC - it will still be created as long as Provisioner has access to do it.** +**Provisioner Controller ignores RBAC. If the user creating the Pod didn't had permissions to create PVC - it will still be created as long as Provisioner has access to do it. Please, use admission controllers.** **Releaser Controller is by design automatically makes PVs with `reclaimPolicy: Retain` available to be reclaimed by other consumers without cleaning up any data. Use it with caution - this behavior might not be desirable in most cases. Any data left on the PV after the previous consumer will be available to all the following consumers. You may want to use StatefulSets instead. This controller might be ideal for something like build cache - insensitive data by design required to be shared among different consumers. There is many use cases for this, one of them is documented in [examples/jenkins-kubernetes-plugin-with-build-cache](examples/jenkins-kubernetes-plugin-with-build-cache).** @@ -46,7 +45,7 @@ The problem statement for creating this was - I need a pool of CI/CD build cache ### Why `reclaimPolicy: Retain` is not enough? -When PVC that were using PV with `reclaimPolicy: Retain` is deleted - Kubernetes marks this PV `Released`. Fortunately, this does not let any other PVC to start using it. I say fortunately because imagine if it did - meaning all the data on the volume could be accessed by a new consumer. This is not what `reclaimPolicy: Retain` is designed for - it only allows cluster administrators to recover the data after accidental PVC deletion. Even now deprecated `reclaimPolicy: Recycle` was performing a cleanup before making PV `Available` again. Unfortunately, this just doesn't work for something like a CI/CD build cache, where you intentionally want to reuse data from the previous consumer. +When PVC that were using PV with `reclaimPolicy: Retain` is deleted - Kubernetes marks this PV `Released`. Fortunately, this will not let any other PVC to use it. I say fortunately because imagine if it did - meaning all the data on the volume could be accessed by a new consumer. This is not what `reclaimPolicy: Retain` is designed for - it only allows cluster administrators to recover the data after accidental PVC deletion. Even now deprecated `reclaimPolicy: Recycle` was performing a cleanup before making PV `Available` again. Unfortunately, this just doesn't work for something like a CI/CD build cache, where you intentionally want to reuse data from the previous consumer. ### Why not a static PVC? @@ -54,7 +53,7 @@ One way to approach this problem statement would be just to create a static PVC ### Why not StatefulSets? -StatefulSets are idiomatic way to reuse PVs and preserve data in Kubernetes. It works great for most of the stateful workload types - unfortunately it doesn't suit very well for CI/CD. Build pods are most likely dynamically generated in CI/CD, each pod is crafted for a specific project, with containers to bring in tools that are needed for this specific project. A simple example - one project might need a MySQL container for its unit tests while another might need a PostgreSQL container - but both are Maven projects so both need a Maven cache. You can't do this with StatefulSets where all the pods are exactly the same. +StatefulSets are idiomatic way to reuse PVs and preserve data in Kubernetes. It works great for most of the stateful workload types - unfortunately it doesn't fit very well for CI/CD. Build pods are most likely dynamically generated in CI/CD, each pod is crafted for a specific project, with containers to bring in tools that are needed for this specific project. A simple example - one project might need a MySQL container for its unit tests while another might need a PostgreSQL container - but both are Maven projects so both need a Maven cache. You can't do this with StatefulSets where all the pods are exactly the same. ### But why dynamic PVC provisioning from the pod annotations? @@ -172,30 +171,21 @@ dynamic-pvc-provisioner \ ## PV Releaser Controller -For Releaser to be able to make PVs claimed by Provisioner `Available` after PVC is gone - Releaser and Provisioner must share the same Controller ID. +For Releaser to be able to make PVs claimed by Provisioner `Available` after PVC is gone - Provisioner must be using Storage Class associated with a Releaser. ### Associate -Once `Released` - PVs doesn't have any indication that they were once associated with a PVC that had association with this Controller ID. To establish this relation - we must catch it while PVC still exists and mark it with our label. If Releaser was down the whole time PVC existed - PV could never be associated making it now orphaned and it will stay as `Released` - Releaser can't know it have to make it `Available`. - -Releaser listens for PV creations/updates. -The following conditions must be met for a PV to be associated with a Releaser: - -- PV doesn't already have `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` association. -- `spec.claimRef` must refer to a PVC that either has `metadata.labels."dynamic-pvc-provisioner.kubernetes.io/managed-by"` or `reclaimable-pv-releaser.kubernetes.io/managed-by` set to this Controller ID. If both labels are set - both should point to this Controller ID. -- `--disable-automatic-association` must be `false`. - -To establish association Releaser will set itself to `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` on this PV. +Releaser listens for Storage Classes and remembers these that are annotated with `metadata.annotations."reclaimable-pv-releaser.kubernetes.io/controller-id"` pointing to this `-controller-id`. Any PV that is using Storage Class with that annotation is considered associated. ### Release Releaser watches for PVs to be released. The following conditions must be met for a PV to be made `Available`: -- `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` must be set to this Controller ID. +- `metadata.annotations."reclaimable-pv-releaser.kubernetes.io/controller-id"` on Storage Class must be set to this Controller ID. - `status.phase` must be `Released`. -If these conditions are met, Releaser will set `spec.claimRef` to `null`. That will make Kubernetes eventually to mark `status.phase` of this PV as `Available` - making other PVCs able to reclaim this PV. Releaser will also delete `metadata.labels."reclaimable-pv-releaser.kubernetes.io/managed-by"` to remove association - the next PVC might be managed by something else. +If these conditions are met, Releaser will set `spec.claimRef` to `null`. That will make Kubernetes eventually to mark `status.phase` of this PV as `Available` - making other PVCs able to reclaim this PV. ### Usage @@ -207,8 +197,6 @@ Usage of reclaimable-pv-releaser: log to standard error as well as files -controller-id string this controller identity name - use the same string for both provisioner and releaser - -disable-automatic-association - disable automatic PV association -kubeconfig string optional, absolute path to the kubeconfig file -lease-lock-id string diff --git a/cmd/reclaimable-pv-releaser/main.go b/cmd/reclaimable-pv-releaser/main.go index 8830f7c..81c87f2 100644 --- a/cmd/reclaimable-pv-releaser/main.go +++ b/cmd/reclaimable-pv-releaser/main.go @@ -2,7 +2,7 @@ package main import ( "context" - "flag" + controller "github.com/plumber-cd/kubernetes-dynamic-reclaimable-pvc-controllers" "github.com/plumber-cd/kubernetes-dynamic-reclaimable-pvc-controllers/releaser" clientset "k8s.io/client-go/kubernetes" @@ -11,9 +11,6 @@ import ( ) func main() { - var disableAutomaticAssociation bool - flag.BoolVar(&disableAutomaticAssociation, "disable-automatic-association", false, "disable automatic PV association") - var c controller.Controller run := func( ctx context.Context, @@ -23,7 +20,7 @@ func main() { namespace string, controllerId string, ) { - c = releaser.New(ctx, client, namespace, controllerId, disableAutomaticAssociation) + c = releaser.New(ctx, client, namespace, controllerId) if err := c.Run(2, stopCh); err != nil { klog.Fatalf("Error running releaser: %s", err.Error()) } diff --git a/controller.go b/controller.go index f9d7a71..6b9b398 100644 --- a/controller.go +++ b/controller.go @@ -286,6 +286,16 @@ func (c *BasicController) Requeue(queue workqueue.RateLimitingInterface, old int c.Enqueue(queue, new) } +func (c *BasicController) Forget(queue workqueue.RateLimitingInterface, obj interface{}) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + queue.Forget(key) +} + func (c *BasicController) RunWorker( name string, queue workqueue.RateLimitingInterface, diff --git a/examples/basic/README.md b/examples/basic/README.md index 8f3993b..5446fd0 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -11,11 +11,18 @@ nerdctl -n k8s.io build -t kubernetes-dynamic-reclaimable-pvc-controllers:dev . 2. Deploy ```bash +# Create SC +kubectl apply -f ./examples/basic/sc.yaml + helm repo add plumber-cd https://plumber-cd.github.io/helm/ helm repo update helm install provisioner plumber-cd/dynamic-pvc-provisioner -f ./examples/basic/values.yaml helm install releaser plumber-cd/reclaimable-pv-releaser -f ./examples/basic/values.yaml +# Or - using local +helm install provisioner ../helm/charts/dynamic-pvc-provisioner -f ./examples/basic/values.yaml +helm install releaser ../helm/charts/reclaimable-pv-releaser -f ./examples/basic/values.yaml + # Check it came up kubectl logs deployment/provisioner-dynamic-pvc-provisioner kubectl logs deployment/releaser-reclaimable-pv-releaser @@ -24,6 +31,12 @@ kubectl logs deployment/releaser-reclaimable-pv-releaser 3. Test ```bash +# Delete SC and see it is forgotten +kubectl delete -f ./examples/basic/sc.yaml +kubectl logs deployment/releaser-reclaimable-pv-releaser +kubectl get events + +# Test provisioner kubectl apply -f ./examples/basic/sc.yaml kubectl apply -f ./examples/basic/pod.yaml @@ -34,9 +47,6 @@ kubectl describe pod pod-with-dynamic-reclaimable-pvc # Check provisioner logs kubectl logs deployment/provisioner-dynamic-pvc-provisioner -# Check releaser logs -kubectl logs deployment/releaser-reclaimable-pv-releaser - # Check PV and PVC kubectl get pv kubectl get pvc @@ -55,7 +65,6 @@ kubectl get pv kubectl apply -f ./examples/basic/pod.yaml kubectl describe pod pod-with-dynamic-reclaimable-pvc kubectl logs deployment/provisioner-dynamic-pvc-provisioner -kubectl logs deployment/releaser-reclaimable-pv-releaser kubectl delete -f ./examples/basic/pod.yaml kubectl logs deployment/releaser-reclaimable-pv-releaser ``` @@ -66,4 +75,5 @@ kubectl logs deployment/releaser-reclaimable-pv-releaser kubectl delete -f ./examples/basic/sc.yaml helm uninstall provisioner helm uninstall releaser +kubectl delete lease provisioner-dynamic-pvc-provisioner releaser-reclaimable-pv-releaser ``` diff --git a/examples/basic/sc.yaml b/examples/basic/sc.yaml index 5341357..a676726 100644 --- a/examples/basic/sc.yaml +++ b/examples/basic/sc.yaml @@ -2,6 +2,8 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: reclaimable-storage-class + annotations: + reclaimable-pv-releaser.kubernetes.io/controller-id: dynamic-reclaimable-pvc-controllers provisioner: rancher.io/local-path reclaimPolicy: Retain volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/examples/jenkins-kubernetes-plugin-with-build-cache/README.md b/examples/jenkins-kubernetes-plugin-with-build-cache/README.md index 37fdfd2..d23abc5 100644 --- a/examples/jenkins-kubernetes-plugin-with-build-cache/README.md +++ b/examples/jenkins-kubernetes-plugin-with-build-cache/README.md @@ -6,6 +6,8 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: jenkins-maven-cache + annotations: + reclaimable-pv-releaser.kubernetes.io/controller-id: dynamic-reclaimable-pvc-controllers provisioner: kubernetes.io/aws-ebs parameters: type: gp2 @@ -16,6 +18,8 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: jenkins-golang-cache + annotations: + reclaimable-pv-releaser.kubernetes.io/controller-id: dynamic-reclaimable-pvc-controllers provisioner: kubernetes.io/aws-ebs parameters: type: gp2 diff --git a/releaser/releaser.go b/releaser/releaser.go index cad9941..39cffb0 100644 --- a/releaser/releaser.go +++ b/releaser/releaser.go @@ -4,17 +4,20 @@ package releaser import ( "context" "fmt" + "sync" "time" controller "github.com/plumber-cd/kubernetes-dynamic-reclaimable-pvc-controllers" - "github.com/plumber-cd/kubernetes-dynamic-reclaimable-pvc-controllers/provisioner" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" + storagelisters "k8s.io/client-go/listers/storage/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -23,19 +26,20 @@ import ( const ( AgentName = "reclaimable-pv-releaser" - LabelBaseName = AgentName + ".kubernetes.io" - LabelManagedByKey = "managed-by" - LabelManagedBy = AgentName + "/" + LabelManagedByKey + AnnotationBaseName = AgentName + ".kubernetes.io" + AnnotationControllerIdKey = "controller-id" + AnnotationControllerId = AnnotationBaseName + "/" + AnnotationControllerIdKey - Associated = "Associated" - MessagePVAssociated = "PV associated successfully" + SCAdded = "Added" + MessageSCAdded = "SC tracking added" + SCRemoved = "Removed" + MessageSCRemoved = "SC tracking removed" + SCLost = "Lost" + MessageSCLost = "SC tracking removed (lost)" Released = "Released" MessagePVReleased = "PV released successfully" - MessageAssociatePV = "error associating PV %s: %s" - ErrAssociatePV = "ErrAssociatePV" - MessageReleasePV = "error releasing PV %s: %s" ErrReleasePV = "ErrReleasePV" ) @@ -43,11 +47,16 @@ const ( type Releaser struct { controller.BasicController - DisableAutomaticAssociation bool + SCLister storagelisters.StorageClassLister + SCSynced cache.InformerSynced + SCQueue workqueue.RateLimitingInterface PVLister corelisters.PersistentVolumeLister PVSynced cache.InformerSynced PVQueue workqueue.RateLimitingInterface + + managedSCMutex *sync.Mutex + managedSCSet map[string]struct{} } func New( @@ -55,7 +64,6 @@ func New( kubeClientSet kubernetes.Interface, namespace, controllerId string, - disableAutomaticAssociation bool, ) controller.Controller { klog.Info("Releaser starting...") @@ -63,23 +71,45 @@ func New( klog.Warningf("Releaser can't run within a namespace as PVs are not namespaced resources - ignoring -namespace=%s and acting in the scope of the cluster", namespace) } - if disableAutomaticAssociation { - klog.Warningf("Automatic PV association is disabled - make sure you label PV manually with '%s: %s' label", LabelManagedBy, controllerId) - } - c := controller.New(ctx, kubeClientSet, "", AgentName, controllerId) + scInformer := c.KubeInformerFactory.Storage().V1().StorageClasses() pvInformer := c.KubeInformerFactory.Core().V1().PersistentVolumes() r := &Releaser{ - BasicController: *c, - DisableAutomaticAssociation: disableAutomaticAssociation, - PVLister: pvInformer.Lister(), - PVSynced: pvInformer.Informer().HasSynced, - PVQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "PersistentVolumes"), + BasicController: *c, + + SCLister: scInformer.Lister(), + SCSynced: scInformer.Informer().HasSynced, + SCQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"), + + PVLister: pvInformer.Lister(), + PVSynced: pvInformer.Informer().HasSynced, + PVQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "PersistentVolumes"), + + managedSCMutex: &sync.Mutex{}, + managedSCSet: make(map[string]struct{}), } klog.V(2).Info("Setting up event handlers") + + scInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + r.Enqueue(r.SCQueue, obj) + }, + UpdateFunc: func(old, new interface{}) { + r.Requeue(r.SCQueue, old, new) + }, + DeleteFunc: func(obj interface{}) { + if sc, ok := obj.(*v1.StorageClass); ok { + r.Forget(r.SCQueue, obj) + r.removeManagedSC(sc) + return + } + klog.Warningf("Received DeleteFunc on %T - skip", obj) + }, + }) + pvInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { r.Enqueue(r.PVQueue, obj) @@ -98,11 +128,21 @@ func (r *Releaser) Run(threadiness int, stopCh <-chan struct{}) error { stopCh, func(threadiness int, stopCh <-chan struct{}) error { klog.V(2).Info("Waiting for informer caches to sync") + + if ok := cache.WaitForCacheSync(stopCh, r.SCSynced); !ok { + return fmt.Errorf("failed to wait for SC caches to sync") + } + if ok := cache.WaitForCacheSync(stopCh, r.PVSynced); !ok { - return fmt.Errorf("failed to wait for caches to sync") + return fmt.Errorf("failed to wait for PV caches to sync") } klog.V(2).Info("Starting workers") + go wait.Until( + r.RunWorker("sc", r.SCQueue, r.scSyncHandler), + time.Second, + stopCh, + ) for i := 0; i < threadiness; i++ { go wait.Until( r.RunWorker("pv", r.PVQueue, r.pvSyncHandler), @@ -111,9 +151,32 @@ func (r *Releaser) Run(threadiness int, stopCh <-chan struct{}) error { ) } + preExistedSC, err := r.SCLister.List(labels.Everything()) + if err != nil { + // If we can't list pre-existent objects - that would be broken state + // It is better to fail fast, this is not expected condition + panic(err) + } + for _, sc := range preExistedSC { + r.Enqueue(r.SCQueue, sc) + } + + preExistedPV, err := r.PVLister.List(labels.Everything()) + if err != nil { + // If we can't list pre-existent objects - that would be broken state + // It is better to fail fast, this is not expected condition + panic(err) + } + for _, pv := range preExistedPV { + r.Enqueue(r.PVQueue, pv) + } + return nil }, func() { + if r.SCQueue != nil { + r.SCQueue.ShutDown() + } if r.PVQueue != nil { r.PVQueue.ShutDown() } @@ -127,86 +190,86 @@ func (r *Releaser) Stop() { klog.Info("Releaser stopped") } -func (r *Releaser) pvSyncHandler(_, name string) error { - pv, err := r.PVLister.Get(name) +func (r *Releaser) addManagedSC(sc *v1.StorageClass) { + r.managedSCMutex.Lock() + defer r.managedSCMutex.Unlock() + if _, exists := r.managedSCSet[sc.ObjectMeta.Name]; exists { + return + } + r.managedSCSet[sc.ObjectMeta.Name] = struct{}{} + r.Recorder.Event(sc, corev1.EventTypeNormal, SCAdded, MessageSCAdded) +} + +func (r *Releaser) removeManagedSC(sc *v1.StorageClass) { + r.managedSCMutex.Lock() + defer r.managedSCMutex.Unlock() + if _, exists := r.managedSCSet[sc.ObjectMeta.Name]; !exists { + return + } + delete(r.managedSCSet, sc.ObjectMeta.Name) + r.Recorder.Event(sc, corev1.EventTypeNormal, SCRemoved, MessageSCRemoved) +} + +func (r *Releaser) removeMissingSC(name string) { + r.managedSCMutex.Lock() + defer r.managedSCMutex.Unlock() + if _, exists := r.managedSCSet[name]; !exists { + return + } + delete(r.managedSCSet, name) + r.Recorder.Event(&corev1.ObjectReference{ + APIVersion: "storage.k8s.io/v1", + Kind: "StorageClass", + Name: name, + }, corev1.EventTypeNormal, SCLost, MessageSCLost) +} + +func (r *Releaser) scSyncHandler(_, name string) error { + sc, err := r.SCLister.Get(name) if err != nil { if errors.IsNotFound(err) { utilruntime.HandleError( - fmt.Errorf("pv '%s' in work queue no longer exists", name), + fmt.Errorf("sc '%s' in work queue no longer exists", name), ) + r.removeMissingSC(name) return nil } return err } - manager, ok := pv.ObjectMeta.Labels[LabelManagedBy] + manager, ok := sc.ObjectMeta.Annotations[AnnotationControllerId] if ok { if manager == r.ControllerId { - return r.pvReleaseHandler(pv) + r.addManagedSC(sc) + return nil } - klog.V(5).Infof("PV %s is managed by '%s', not me '%s', skip", pv.ObjectMeta.Name, manager, r.ControllerId) - } else if !r.DisableAutomaticAssociation { - return r.pvAssociateHandler(pv) + klog.V(5).Infof("SC %s is not annotated with '%s=%s', skip", sc.ObjectMeta.Name, AnnotationControllerId, r.ControllerId) } return nil } -func (r *Releaser) pvAssociateHandler(pv *corev1.PersistentVolume) error { - if pv.Spec.ClaimRef == nil { - klog.V(5).Infof("PV %s had no claim ref, skip", pv.ObjectMeta.Name) - return nil - } - - pvc, err := r.KubeClientSet.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace). - Get(r.Ctx, pv.Spec.ClaimRef.Name, metav1.GetOptions{}) +func (r *Releaser) pvSyncHandler(_, name string) error { + pv, err := r.PVLister.Get(name) if err != nil { if errors.IsNotFound(err) { - klog.V(5).Infof("PV %s had claim ref to the void, skip", pv.ObjectMeta.Name) + utilruntime.HandleError( + fmt.Errorf("pv '%s' in work queue no longer exists", name), + ) return nil } return err } - pvcReleaserOwner, releaserOwnerOk := pvc.ObjectMeta.Labels[LabelManagedBy] - pvcProvisionerOwner, provisionerOwnerOk := pvc.ObjectMeta.Labels[provisioner.LabelManagedBy] - if !provisionerOwnerOk && !releaserOwnerOk { - klog.V(5).Infof("PVC has no manager, skip PV %s", pvc.ObjectMeta.Name, pv.ObjectMeta.Name) - return nil - } else if releaserOwnerOk && pvcReleaserOwner != r.ControllerId { - klog.V(5).Infof("PVC %s is managed by releaser '%s', not me '%s', skip PV %s", pvc.ObjectMeta.Name, pvcReleaserOwner, r.ControllerId, pv.ObjectMeta.Name) - return nil - } else if provisionerOwnerOk && pvcProvisionerOwner != r.ControllerId { - klog.V(5).Infof("PVC %s is managed by provisioner '%s', not me '%s', skip PV %s", pvc.ObjectMeta.Name, pvcProvisionerOwner, r.ControllerId, pv.ObjectMeta.Name) - return nil - } - - klog.V(4).Infof("PV %s is matched for association based on PVC %s", pv.ObjectMeta.Name, pvc.ObjectMeta.Name) - - pvCopy := pv.DeepCopy() - if pvCopy.ObjectMeta.Labels == nil { - pvCopy.ObjectMeta.Labels = make(map[string]string) - } - pvCopy.ObjectMeta.Labels[LabelManagedBy] = r.ControllerId - _, err = r.KubeClientSet.CoreV1().PersistentVolumes().Update(r.Ctx, pvCopy, metav1.UpdateOptions{}) - if err != nil { - if errors.IsConflict(err) { - klog.V(4).Infof("PV %s had a conflict - ignore it, it will be queued again with a new version", pv.ObjectMeta.Name) - return nil - } - - r.Recorder.Event( - pvCopy, - corev1.EventTypeWarning, - ErrAssociatePV, - fmt.Sprintf(MessageAssociatePV, pvCopy, err), - ) - return err + _, ok := r.managedSCSet[pv.Spec.StorageClassName] + if ok { + return r.pvReleaseHandler(pv) + } else { + klog.V(5).Infof("SC %q for PV %q is not associated with this controller ID %q, skip", pv.Spec.StorageClassName, pv.ObjectMeta.Name, r.ControllerId) } - r.Recorder.Event(pv, corev1.EventTypeNormal, Associated, MessagePVAssociated) return nil } @@ -218,10 +281,6 @@ func (r *Releaser) pvReleaseHandler(pv *corev1.PersistentVolume) error { pvCopy := pv.DeepCopy() pvCopy.Spec.ClaimRef = nil - if !r.DisableAutomaticAssociation { - klog.V(4).Infof("Removing PV %s association with myself ('%s')", pv.ObjectMeta.Name, r.ControllerId) - delete(pvCopy.ObjectMeta.Labels, LabelManagedBy) - } _, err := r.KubeClientSet.CoreV1().PersistentVolumes().Update(r.Ctx, pvCopy, metav1.UpdateOptions{}) if err != nil { if errors.IsConflict(err) {