diff --git a/test/e2e/framework/clusterresourcebinding.go b/test/e2e/framework/clusterresourcebinding.go index f8aa53bdc936..119e51ba3612 100644 --- a/test/e2e/framework/clusterresourcebinding.go +++ b/test/e2e/framework/clusterresourcebinding.go @@ -18,7 +18,9 @@ package framework import ( "context" + "fmt" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,3 +38,11 @@ func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fi return fit(clusterResourceBinding) }, pollTimeout, pollInterval).Should(gomega.Equal(true)) } + +// RemoveClusterResourceBinding delete ClusterResourceBinding with karmada client. +func RemoveClusterResourceBinding(client karmada.Interface, name string) { + ginkgo.By(fmt.Sprintf("Removing ClusterResourceBinding(%s)", name), func() { + err := client.WorkV1alpha2().ClusterResourceBindings().Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} diff --git a/test/e2e/framework/resourcebinding.go b/test/e2e/framework/resourcebinding.go index de2d6386f06f..25adfe590ba9 100644 --- a/test/e2e/framework/resourcebinding.go +++ b/test/e2e/framework/resourcebinding.go @@ -80,3 +80,11 @@ func WaitGracefulEvictionTasksDone(client karmada.Interface, namespace, name str }, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred()) }) } + +// RemoveResourceBinding delete ResourceBinding with karmada client. +func RemoveResourceBinding(client karmada.Interface, namespace, name string) { + ginkgo.By(fmt.Sprintf("Removing ResourceBinding(%s/%s)", namespace, name), func() { + err := client.WorkV1alpha2().ResourceBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} diff --git a/test/e2e/scheduling_test.go b/test/e2e/scheduling_test.go index 8eb7c40f0042..1ef67abf037b 100644 --- a/test/e2e/scheduling_test.go +++ b/test/e2e/scheduling_test.go @@ -696,6 +696,130 @@ var _ = ginkgo.Describe("[JobReplicaScheduling] JobReplicaSchedulingStrategy tes }) }) +var _ = ginkgo.Describe("[Suspension] ResourceBinding testing", func() { + var deploymentName string + var rb *workv1alpha2.ResourceBinding + + ginkgo.BeforeEach(func() { + // Create deployment. + deploymentName = deploymentNamePrefix + rand.String(RandomStrLength) + deployment := helper.NewDeployment(testNamespace, deploymentName) + framework.CreateDeployment(kubeClient, deployment) + _, err := kubeClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get deployment") + + // Create ResourceBinding. + rb = helper.NewResourceBinding(deploymentName, workv1alpha2.ObjectReference{ + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Namespace: testNamespace, + Name: deploymentName, + }, &policyv1alpha1.Placement{ + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDivided, + ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted, + }, + }, true) + rb, err = karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Create(context.TODO(), rb, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create ResourceBinding") + }) + + ginkgo.AfterEach(func() { + framework.RemoveResourceBinding(karmadaClient, testNamespace, rb.Name) + framework.RemoveDeployment(kubeClient, testNamespace, deploymentName) + }) + + ginkgo.It("should not be scheduled when suspended and should be scheduled after resumed for resourceBinding", func() { + ginkgo.By("Waiting for the ResourceBinding not to be scheduled when suspended", func() { + gomega.Eventually(func() bool { + latestRB, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return meta.IsStatusConditionTrue(latestRB.Status.Conditions, workv1alpha2.Scheduled) + }, pollTimeout, pollInterval).Should(gomega.BeFalse(), "ResourceBinding should not be scheduled when suspended") + }) + + ginkgo.By("Resuming the ResourceBinding", func() { + gomega.Eventually(func() bool { + newResourceBinding, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + newResourceBinding.Spec.Suspension = nil + _, err = karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Update(context.TODO(), newResourceBinding, metav1.UpdateOptions{}) + return err == nil + }, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ResourceBinding should can be resumed") + }) + + ginkgo.By("Waiting for the ResourceBinding to be scheduled after resumed", func() { + gomega.Eventually(func() bool { + latestRB, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return meta.IsStatusConditionTrue(latestRB.Status.Conditions, workv1alpha2.Scheduled) + }, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ResourceBinding should be scheduled after resumed") + }) + }) +}) + +var _ = ginkgo.Describe("[Suspension] ClusterResourceBinding testing", func() { + var configMapName string + var crb *workv1alpha2.ClusterResourceBinding + + ginkgo.BeforeEach(func() { + // Create ConfigMap. + configMapName = configMapNamePrefix + rand.String(RandomStrLength) + configMap := helper.NewConfigMap(testNamespace, configMapName, make(map[string]string)) + framework.CreateConfigMap(kubeClient, configMap) + _, err := kubeClient.CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get ConfigMap") + + // Create ClusterResourceBinding. + targetMember := framework.ClusterNames()[0] + crb = helper.NewClusterResourceBinding(configMapName, workv1alpha2.ObjectReference{ + APIVersion: configMap.APIVersion, + Kind: configMap.Kind, + Namespace: testNamespace, + Name: configMapName, + }, &policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetMember}, + }, + }, true) + crb, err = karmadaClient.WorkV1alpha2().ClusterResourceBindings().Create(context.TODO(), crb, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create ClusterResourceBinding") + }) + + ginkgo.AfterEach(func() { + framework.RemoveClusterResourceBinding(karmadaClient, crb.Name) + framework.RemoveConfigMap(kubeClient, testNamespace, configMapName) + }) + + ginkgo.It("should not be scheduled when suspended and should be scheduled after resumed for clusterResourceBinding", func() { + ginkgo.By("Waiting for the ClusterResourceBinding not to be scheduled when suspended", func() { + gomega.Eventually(func() bool { + latestCRB, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return meta.IsStatusConditionTrue(latestCRB.Status.Conditions, workv1alpha2.Scheduled) + }, pollTimeout, pollInterval).Should(gomega.BeFalse(), "ClusterResourceBinding should not be scheduled when suspended") + }) + + ginkgo.By("Resuming the ClusterResourceBinding", func() { + gomega.Eventually(func() bool { + newClusterResourceBinding, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + newClusterResourceBinding.Spec.Suspension = nil + _, err = karmadaClient.WorkV1alpha2().ClusterResourceBindings().Update(context.TODO(), newClusterResourceBinding, metav1.UpdateOptions{}) + return err == nil + }, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ClusterResourceBinding should can be resumed") + }) + + ginkgo.By("Waiting for the ClusterResourceBinding to be scheduled after resumed", func() { + gomega.Eventually(func() bool { + latestCRB, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + return meta.IsStatusConditionTrue(latestCRB.Status.Conditions, workv1alpha2.Scheduled) + }, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ClusterResourceBinding should be scheduled after resumed") + }) + }) +}) + // get the resource binding associated with the workload func getResourceBinding(workload interface{}) (*workv1alpha2.ResourceBinding, error) { obj, err := utilhelper.ToUnstructured(workload) diff --git a/test/helper/resource.go b/test/helper/resource.go index 5f001efa7118..730ff228d8e9 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + uuid "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/utils/ptr" workloadv1alpha1 "github.com/karmada-io/karmada/examples/customresourceinterpreter/apis/workload/v1alpha1" @@ -40,7 +41,9 @@ import ( autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" networkingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" ) // These are different resource units. @@ -993,3 +996,49 @@ func NewWorkloadRebalancer(name string, objectReferences []appsv1alpha1.ObjectRe }, } } + +// NewResourceBinding will build a new resourceBinding object. +func NewResourceBinding(name string, resource workv1alpha2.ObjectReference, placement *policyv1alpha1.Placement, suspended bool) *workv1alpha2.ResourceBinding { + rb := &workv1alpha2.ResourceBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: workv1alpha2.ResourceKindResourceBinding, + APIVersion: workv1alpha2.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + policyv1alpha1.PropagationPolicyPermanentIDLabel: string(uuid.NewUUID()), + }, + }, + Spec: workv1alpha2.ResourceBindingSpec{ + SchedulerName: "default-scheduler", + Resource: resource, + Placement: placement, + Suspension: &workv1alpha2.Suspension{Scheduling: &suspended}, + }, + } + return rb +} + +// NewClusterResourceBinding will build a new resourceBinding object. +func NewClusterResourceBinding(name string, resource workv1alpha2.ObjectReference, placement *policyv1alpha1.Placement, suspended bool) *workv1alpha2.ClusterResourceBinding { + crb := &workv1alpha2.ClusterResourceBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: workv1alpha2.ResourceKindClusterResourceBinding, + APIVersion: workv1alpha2.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + policyv1alpha1.PropagationPolicyPermanentIDLabel: string(uuid.NewUUID()), + }, + }, + Spec: workv1alpha2.ResourceBindingSpec{ + SchedulerName: "default-scheduler", + Resource: resource, + Placement: placement, + Suspension: &workv1alpha2.Suspension{Scheduling: &suspended}, + }, + } + return crb +}