feat: cluster-level resource scheduling suspend and resume capabiliti…

…es (#5937) * RB suspension: API change Signed-off-by: Monokaix <[email protected]> * Feat: scheduler supports binging suspension Signed-off-by: Monokaix <[email protected]> --------- Signed-off-by: Monokaix <[email protected]>
karmada-io · Jan 2, 2025 · 7a85687 · 7a85687
1 parent ba1e68d
commit 7a85687
Show file tree

Hide file tree

Showing 10 changed files with 215 additions and 14 deletions.
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -20435,7 +20435,7 @@
       }
     },
     "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.Suspension": {
-      "description": "Suspension defines the policy for suspending of propagation.",
+      "description": "Suspension defines the policy for suspending dispatching and scheduling.",
       "type": "object",
       "properties": {
         "dispatching": {
@@ -20445,6 +20445,10 @@
         "dispatchingOnClusters": {
           "description": "DispatchingOnClusters declares a list of clusters to which the dispatching should be suspended. Note: Can not co-exist with Dispatching which is used to suspend all.",
           "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.policy.v1alpha1.SuspendClusters"
+        },
+        "scheduling": {
+          "description": "Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not process resource binding when the value is true and resume scheduling when it's false or nil. This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling manage resource allocation, prioritize critical workloads, etc. It is expected that third-party systems use an admission webhook to suspend scheduling at the time of ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may lead to ineffective suspension.",
+          "type": "boolean"
         }
       }
     },

diff --git a/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml b/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml
@@ -1281,6 +1281,16 @@ spec:
                           type: string
                         type: array
                     type: object
+                  scheduling:
+                    description: |-
+                      Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+                      process resource binding when the value is true and resume scheduling when it's false or nil.
+                      This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+                      manage resource allocation, prioritize critical workloads, etc.
+                      It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+                      ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+                      lead to ineffective suspension.
+                    type: boolean
                 type: object
             required:
             - resource

diff --git a/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml b/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml
@@ -1281,6 +1281,16 @@ spec:
                           type: string
                         type: array
                     type: object
+                  scheduling:
+                    description: |-
+                      Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+                      process resource binding when the value is true and resume scheduling when it's false or nil.
+                      This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+                      manage resource allocation, prioritize critical workloads, etc.
+                      It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+                      ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+                      lead to ineffective suspension.
+                    type: boolean
                 type: object
             required:
             - resource

diff --git a/pkg/apis/work/v1alpha2/binding_types.go b/pkg/apis/work/v1alpha2/binding_types.go
@@ -322,9 +322,19 @@ type BindingSnapshot struct {
 	Clusters []TargetCluster `json:"clusters,omitempty"`
 }
 
-// Suspension defines the policy for suspending of propagation.
+// Suspension defines the policy for suspending dispatching and scheduling.
 type Suspension struct {
 	policyv1alpha1.Suspension `json:",inline"`
+
+	// Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+	// process resource binding when the value is true and resume scheduling when it's false or nil.
+	// This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+	// manage resource allocation, prioritize critical workloads, etc.
+	// It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+	// ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+	// lead to ineffective suspension.
+	// +optional
+	Scheduling *bool `json:"scheduling,omitempty"`
 }
 
 // ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources.

diff --git a/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/work/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go
diff --git a/pkg/scheduler/event_handler.go b/pkg/scheduler/event_handler.go
@@ -101,10 +101,16 @@ func (s *Scheduler) resourceBindingEventFilter(obj interface{}) bool {
 		if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
 			return false
 		}
+		if util.IsBindingSuspendScheduling(t) {
+			return false
+		}
 	case *workv1alpha2.ClusterResourceBinding:
 		if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
 			return false
 		}
+		if util.IsClusterBindingSuspendScheduling(t) {
+			return false
+		}
 	}
 
 	return util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.PropagationPolicyPermanentIDLabel) != "" ||

diff --git a/pkg/scheduler/event_handler_test.go b/pkg/scheduler/event_handler_test.go
@@ -24,6 +24,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/utils/ptr"
 
 	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
 	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
@@ -40,65 +41,65 @@ func TestResourceBindingEventFilter(t *testing.T) {
 		{
 			name:           "ResourceBinding: Matching scheduler name, no labels",
 			schedulerName:  "test-scheduler",
-			obj:            createResourceBinding("test-rb", "test-scheduler", nil),
+			obj:            createResourceBinding("test-rb", "test-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ResourceBinding: Non-matching scheduler name",
 			schedulerName:  "test-scheduler",
-			obj:            createResourceBinding("test-rb", "other-scheduler", nil),
+			obj:            createResourceBinding("test-rb", "other-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:          "ResourceBinding: Matching scheduler name, with PropagationPolicyPermanentIDLabel",
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.PropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
 			name:          "ResourceBinding: Matching scheduler name, with ClusterPropagationPolicyPermanentIDLabel",
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
 			name:          "ResourceBinding: Matching scheduler name, with BindingManagedByLabel",
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				workv1alpha2.BindingManagedByLabel: "test-manager",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
 			name:          "ResourceBinding: Matching scheduler name, with empty PropagationPolicyPermanentIDLabel",
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.PropagationPolicyPermanentIDLabel: "",
-			}),
+			}, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ClusterResourceBinding: Matching scheduler name, no labels",
 			schedulerName:  "test-scheduler",
-			obj:            createClusterResourceBinding("test-crb", "test-scheduler", nil),
+			obj:            createClusterResourceBinding("test-crb", "test-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ClusterResourceBinding: Non-matching scheduler name",
 			schedulerName:  "test-scheduler",
-			obj:            createClusterResourceBinding("test-crb", "other-scheduler", nil),
+			obj:            createClusterResourceBinding("test-crb", "other-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:          "ClusterResourceBinding: Matching scheduler name, with ClusterPropagationPolicyPermanentIDLabel",
 			schedulerName: "test-scheduler",
 			obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
 				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
@@ -113,6 +114,22 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			obj:            "not-a-valid-object",
 			expectedResult: false,
 		},
+		{
+			name:          "ResourceBinding suspended",
+			schedulerName: "test-scheduler",
+			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
+				workv1alpha2.BindingManagedByLabel: "test-manager",
+			}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
+			expectedResult: false,
+		},
+		{
+			name:          "ClusterResourceBinding suspended",
+			schedulerName: "test-scheduler",
+			obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
+				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
+			}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
+			expectedResult: false,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -404,26 +421,28 @@ func createCluster(name string, generation int64, labels map[string]string) *clu
 	}
 }
 
-func createResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ResourceBinding {
+func createResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ResourceBinding {
 	return &workv1alpha2.ResourceBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
 			Labels: labels,
 		},
 		Spec: workv1alpha2.ResourceBindingSpec{
 			SchedulerName: schedulerName,
+			Suspension:    suspension,
 		},
 	}
 }
 
-func createClusterResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ClusterResourceBinding {
+func createClusterResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ClusterResourceBinding {
 	return &workv1alpha2.ClusterResourceBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
 			Labels: labels,
 		},
 		Spec: workv1alpha2.ResourceBindingSpec{
 			SchedulerName: schedulerName,
+			Suspension:    suspension,
 		},
 	}
 }

diff --git a/pkg/util/binding.go b/pkg/util/binding.go
@@ -110,3 +110,19 @@ func RescheduleRequired(rescheduleTriggeredAt, lastScheduledTime *metav1.Time) b
 	}
 	return rescheduleTriggeredAt.After(lastScheduledTime.Time)
 }
+
+// IsBindingSuspendScheduling tells whether resource binding is scheduling suspended.
+func IsBindingSuspendScheduling(rb *workv1alpha2.ResourceBinding) bool {
+	if rb == nil || rb.Spec.Suspension == nil || rb.Spec.Suspension.Scheduling == nil {
+		return false
+	}
+	return *rb.Spec.Suspension.Scheduling
+}
+
+// IsClusterBindingSuspendScheduling tells whether cluster resource binding is scheduling suspended.
+func IsClusterBindingSuspendScheduling(crb *workv1alpha2.ClusterResourceBinding) bool {
+	if crb == nil || crb.Spec.Suspension == nil || crb.Spec.Suspension.Scheduling == nil {
+		return false
+	}
+	return *crb.Spec.Suspension.Scheduling
+}