fix(region): vendor update for region sync (#19863)

go.mod

@@ -89,7 +89,7 @@ require (
 	k8s.io/cluster-bootstrap v0.19.3
 	k8s.io/cri-api v0.22.17
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402105128-a5d772986f23
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402114032-4a21d386641e
 	yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
 	yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91

go.sum

@@ -1210,8 +1210,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402105128-a5d772986f23 h1:F9bFu8UuGi2TK4QHGqFMeT9lc+9Y8hcWYar8p3YyYqM=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402105128-a5d772986f23/go.mod h1:dsUESXIbXJ+/ywbNClhldOrbPOiBi2udrgOnB/ffoWk=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402114032-4a21d386641e h1:5IlhObq0+F/+dj6NsATXtXh3nXj+PwM1HJ8+DwMCFS8=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402114032-4a21d386641e/go.mod h1:dsUESXIbXJ+/ywbNClhldOrbPOiBi2udrgOnB/ffoWk=
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=

pkg/cloudcommon/db/tablespec.go

@@ -305,7 +305,6 @@ func (ts *sTableSpec) informUpdate(ctx context.Context, dt interface{}, oldObj *
 			debug.PrintStack()
 			return
 		}
-		debug.PrintStack()
 		if err := informer.Update(ctx, obj, oldObj); err != nil {
 			if errors.Cause(err) == informer.ErrBackendNotInit {
 				log.V(4).Warningf("informer backend not init")

pkg/compute/models/cloudproviders.go

@@ -1541,7 +1541,7 @@ func (provider *SCloudprovider) prepareCloudproviderRegions(ctx context.Context,
 		return []SCloudproviderregion{*cpr}, nil
 	}
 	iregions := driver.GetIRegions()
-	externalIdPrefix := driver.GetCloudRegionExternalIdPrefix()
+	externalIdPrefix := strings.TrimSuffix(driver.GetCloudRegionExternalIdPrefix(), "/")
 	_, _, cprs, result := CloudregionManager.SyncRegions(ctx, userCred, provider, externalIdPrefix, iregions)
 	if result.IsError() {
 		log.Errorf("syncRegion fail %s", result.Result())

pkg/compute/models/cloudregions.go

@@ -17,7 +17,6 @@ package models
 import (
 	"context"
 	"database/sql"
-	"strings"
 	"time"
 
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
@@ -64,7 +63,6 @@ func init() {
 type SCloudregion struct {
 	db.SEnabledStatusStandaloneResourceBase
 	SI18nResourceBase
-	SManagedResourceBase
 	db.SExternalizedResourceBase
 
 	cloudprovider.SGeographicInfo
@@ -473,16 +471,15 @@ func (self *SCloudregion) GetServerSkus() ([]SServerSku, error) {
 }
 
 func (self *SCloudprovider) GetRegionByExternalIdPrefix(prefix string) ([]SCloudregion, error) {
-	factory, err := self.GetProviderFactory()
-	if err != nil {
-		return nil, err
-	}
 	regions := make([]SCloudregion, 0)
-	q := CloudregionManager.Query().Startswith("external_id", prefix)
-	if !factory.IsPublicCloud() && !strings.Contains(prefix, "/") {
-		q = CloudregionManager.Query().Equals("manager_id", self.Id)
-	}
-	err = db.FetchModelObjects(CloudregionManager, q, &regions)
+	q := CloudregionManager.Query()
+	q = q.Filter(
+		sqlchemy.OR(
+			sqlchemy.Startswith(q.Field("external_id"), prefix),
+			sqlchemy.Equals(q.Field("external_id"), prefix),
+		),
+	)
+	err := db.FetchModelObjects(CloudregionManager, q, &regions)
 	if err != nil {
 		return nil, err
 	}
@@ -494,17 +491,7 @@ func (manager *SCloudregionManager) GetRegionByProvider(provider string) ([]SClo
 	q := manager.Query().Equals("provider", provider)
 	err := db.FetchModelObjects(manager, q, &regions)
 	if err != nil {
-		log.Errorf("%s", err)
-		return nil, err
-	}
-	return regions, nil
-}
-
-func (manager *SCloudregionManager) getCloudregionsByProviderId(providerId string) ([]SCloudregion, error) {
-	regions := []SCloudregion{}
-	err := fetchByManagerId(manager, providerId, &regions)
-	if err != nil {
-		return nil, errors.Wrap(err, "fetchByManagerId")
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
 	}
 	return regions, nil
 }
@@ -521,8 +508,8 @@ func (manager *SCloudregionManager) SyncRegions(
 	[]SCloudproviderregion,
 	compare.SyncResult,
 ) {
-	lockman.LockRawObject(ctx, "cloudregions", externalIdPrefix)
-	defer lockman.ReleaseRawObject(ctx, "cloudregions", externalIdPrefix)
+	lockman.LockRawObject(ctx, manager.Keyword(), externalIdPrefix)
+	defer lockman.ReleaseRawObject(ctx, manager.Keyword(), externalIdPrefix)
 
 	syncResult := compare.SyncResult{}
 	localRegions := make([]SCloudregion, 0)
@@ -555,11 +542,10 @@ func (manager *SCloudregionManager) SyncRegions(
 	}
 	for i := 0; i < len(commondb); i += 1 {
 		// update
-		err = commondb[i].syncWithCloudRegion(ctx, userCred, commonext[i], cloudProvider)
+		err = commondb[i].syncWithCloudRegion(ctx, userCred, commonext[i])
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i], false)
 			cpr := CloudproviderRegionManager.FetchByIdsOrCreate(cloudProvider.Id, commondb[i].Id)
 			cpr.setCapabilities(ctx, userCred, commonext[i].GetCapabilities())
 			cloudProviderRegions = append(cloudProviderRegions, *cpr)
@@ -573,7 +559,6 @@ func (manager *SCloudregionManager) SyncRegions(
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i], false)
 			cpr := CloudproviderRegionManager.FetchByIdsOrCreate(cloudProvider.Id, new.Id)
 			cpr.setCapabilities(ctx, userCred, added[i].GetCapabilities())
 			cloudProviderRegions = append(cloudProviderRegions, *cpr)
@@ -592,17 +577,12 @@ func (self *SCloudregion) syncRemoveCloudRegion(ctx context.Context, userCred mc
 	return self.purgeAll(ctx, cloudProvider.Id)
 }
 
-func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mcclient.TokenCredential, cloudRegion cloudprovider.ICloudRegion, provider *SCloudprovider) error {
+func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mcclient.TokenCredential, cloudRegion cloudprovider.ICloudRegion) error {
 	err := CloudregionManager.SyncI18ns(ctx, userCred, self, cloudRegion.GetI18n())
 	if err != nil {
 		return errors.Wrap(err, "SyncI18ns")
 	}
 
-	factory, err := provider.GetProviderFactory()
-	if err != nil {
-		return err
-	}
-
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		if !utils.IsInStringArray(self.Provider, api.PRIVATE_CLOUD_PROVIDERS) {
 			self.Name = cloudRegion.GetName()
@@ -618,10 +598,6 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 
 		self.IsEmulated = cloudRegion.IsEmulated()
 
-		if !factory.IsPublicCloud() && !factory.IsOnPremise() && !factory.IsMultiTenant() {
-			self.ManagerId = provider.Id
-		}
-
 		return nil
 	})
 	if err != nil && errors.Cause(err) != sqlchemy.ErrNoDataToUpdate {
@@ -645,14 +621,7 @@ func (manager *SCloudregionManager) newFromCloudRegion(ctx context.Context, user
 
 	region.IsEmulated = cloudRegion.IsEmulated()
 
-	factory, err := provider.GetProviderFactory()
-	if err != nil {
-		return nil, err
-	}
-	if !factory.IsOnPremise() && !factory.IsPublicCloud() {
-		region.ManagerId = provider.Id
-	}
-
+	var err error
 	err = func() error {
 		lockman.LockRawObject(ctx, manager.Keyword(), "name")
 		defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
@@ -1066,7 +1035,7 @@ func (self *SCloudregion) GetRegionExtId() string {
 }
 
 func (self *SCloudregion) ValidateUpdateCondition(ctx context.Context) error {
-	if len(self.ExternalId) > 0 && len(self.ManagerId) == 0 {
+	if len(self.ExternalId) > 0 {
 		return httperrors.NewConflictError("Cannot update external resource")
 	}
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateCondition(ctx)
@@ -1334,13 +1303,14 @@ func (self *SCloudregion) StartSyncSkusTask(ctx context.Context, userCred mcclie
 	return task.ScheduleRun(nil)
 }
 
-func (self *SCloudregion) GetCloudprovider() (*SCloudprovider, error) {
-	if len(self.ManagerId) == 0 {
-		return nil, sql.ErrNoRows
-	}
-	provider, err := CloudproviderManager.FetchById(self.ManagerId)
+func (self *SCloudregion) GetCloudproviders() ([]SCloudprovider, error) {
+	sq := CloudproviderRegionManager.Query().Equals("cloudregion_id", self.Id).SubQuery()
+	q := CloudproviderManager.Query()
+	q = q.Join(sq, sqlchemy.Equals(sq.Field("cloudprovider_id"), q.Field("id")))
+	ret := []SCloudprovider{}
+	err := db.FetchModelObjects(CloudproviderManager, q, &ret)
 	if err != nil {
-		return nil, errors.Wrapf(err, "FetchByI(%s)", self.ManagerId)
+		return nil, err
 	}
-	return provider.(*SCloudprovider), nil
+	return ret, nil
 }

pkg/compute/models/server_skus.go

@@ -1460,28 +1460,28 @@ func (self *SServerSku) GetICloudSku(ctx context.Context) (cloudprovider.ICloudS
 		}
 		return nil, errors.Wrapf(err, "GetRegion")
 	}
-	provider, err := region.GetCloudprovider()
+	providers, err := region.GetCloudproviders()
 	if err != nil {
-		if errors.Cause(err) == sql.ErrNoRows {
-			return nil, errors.Wrapf(cloudprovider.ErrNotFound, "GetCloudprovider")
-		}
 		return nil, errors.Wrapf(err, "GetCloudprovider")
 	}
-	driver, err := provider.GetProvider(ctx)
-	if err != nil {
-		return nil, errors.Wrapf(err, "GetDriver()")
-	}
-	iRegion, err := driver.GetIRegionById(region.ExternalId)
-	if err != nil {
-		return nil, errors.Wrapf(err, "GetIRegionById(%s)", region.ExternalId)
-	}
-	skus, err := iRegion.GetISkus()
-	if err != nil {
-		return nil, errors.Wrapf(err, "GetICloudSku")
-	}
-	for i := range skus {
-		if skus[i].GetGlobalId() == self.ExternalId {
-			return skus[i], nil
+	for i := range providers {
+		provider := providers[i]
+		driver, err := provider.GetProvider(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "GetDriver()")
+		}
+		iRegion, err := driver.GetIRegionById(region.ExternalId)
+		if err != nil {
+			return nil, errors.Wrapf(err, "GetIRegionById(%s)", region.ExternalId)
+		}
+		skus, err := iRegion.GetISkus()
+		if err != nil {
+			return nil, errors.Wrapf(err, "GetICloudSku")
+		}
+		for i := range skus {
+			if skus[i].GetGlobalId() == self.ExternalId {
+				return skus[i], nil
+			}
 		}
 	}
 	return nil, errors.Wrapf(cloudprovider.ErrNotFound, self.ExternalId)

pkg/compute/models/vpcs.go

@@ -927,9 +927,6 @@ func (manager *SVpcManager) ValidateCreateData(
 	}
 	region := regionObj.(*SCloudregion)
 	if region.isManaged() {
-		if len(region.ManagerId) > 0 {
-			input.CloudproviderId = region.ManagerId
-		}
 		_, err := validators.ValidateModel(ctx, userCred, CloudproviderManager, &input.CloudproviderId)
 		if err != nil {
 			return input, err

pkg/compute/tasks/server_sku_create_task.go

@@ -51,38 +51,43 @@ func (self *ServerSkuCreateTask) OnInit(ctx context.Context, obj db.IStandaloneM
 		return
 	}
 
-	provider, err := region.GetCloudprovider()
+	providers, err := region.GetCloudproviders()
 	if err != nil {
-		self.taskFail(ctx, sku, errors.Wrapf(err, "GetCloudprovider"))
+		self.taskFail(ctx, sku, errors.Wrapf(err, "GetCloudproviders"))
 		return
 	}
 
-	driver, err := provider.GetProvider(ctx)
-	if err != nil {
-		self.taskFail(ctx, sku, errors.Wrapf(err, "GetDriver"))
-		return
-	}
+	for i := range providers {
+		provider := providers[i]
+		driver, err := provider.GetProvider(ctx)
+		if err != nil {
+			self.taskFail(ctx, sku, errors.Wrapf(err, "GetDriver"))
+			return
+		}
 
-	iRegion, err := driver.GetIRegionById(region.ExternalId)
-	if err != nil {
-		self.taskFail(ctx, sku, errors.Wrapf(err, "GetIRegionById"))
-		return
-	}
+		iRegion, err := driver.GetIRegionById(region.ExternalId)
+		if err != nil {
+			self.taskFail(ctx, sku, errors.Wrapf(err, "GetIRegionById"))
+			return
+		}
 
-	opts := cloudprovider.SServerSkuCreateOption{
-		Name:             sku.Name,
-		CpuCount:         sku.CpuCoreCount,
-		VmemSizeMb:       sku.MemorySizeMB,
-		SysDiskMinSizeGb: sku.SysDiskMinSizeGB,
-		SysDiskMaxSizeGb: sku.SysDiskMaxSizeGB,
-	}
+		opts := cloudprovider.SServerSkuCreateOption{
+			Name:             sku.Name,
+			CpuCount:         sku.CpuCoreCount,
+			VmemSizeMb:       sku.MemorySizeMB,
+			SysDiskMinSizeGb: sku.SysDiskMinSizeGB,
+			SysDiskMaxSizeGb: sku.SysDiskMaxSizeGB,
+		}
 
-	iSku, err := iRegion.CreateISku(&opts)
-	if err != nil {
-		self.taskFail(ctx, sku, errors.Wrapf(err, "CreateISku"))
+		iSku, err := iRegion.CreateISku(&opts)
+		if err != nil {
+			self.taskFail(ctx, sku, errors.Wrapf(err, "CreateISku"))
+			return
+		}
+
+		sku.SyncWithPrivateCloudSku(ctx, self.GetUserCred(), iSku)
+		self.SetStageComplete(ctx, nil)
 		return
 	}
-
-	sku.SyncWithPrivateCloudSku(ctx, self.GetUserCred(), iSku)
-	self.SetStageComplete(ctx, nil)
+	self.taskFail(ctx, sku, errors.Wrapf(cloudprovider.ErrNotFound, "region %s", region.Name))
 }

vendor/modules.txt

@@ -1476,7 +1476,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
-# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402105128-a5d772986f23
+# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240402114032-4a21d386641e
 ## explicit; go 1.18
 yunion.io/x/cloudmux/pkg/apis
 yunion.io/x/cloudmux/pkg/apis/billing