From 3f10e5adad8438affbb00aa776179f17fc781f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 19 Dec 2023 20:26:09 +0800 Subject: [PATCH] fix(region): ignore empty certificate error for ssl sync (#19053) --- go.mod | 2 +- go.sum | 4 +- pkg/compute/models/sslcertificate.go | 4 +- vendor/modules.txt | 2 +- .../pkg/multicloud/huawei/client/client.go | 5 -- .../huawei/client/modules/mod_keypairs.go | 37 --------- .../huawei/client/modules/mod_quotas.go | 37 --------- .../cloudmux/pkg/multicloud/huawei/huawei.go | 20 +---- .../cloudmux/pkg/multicloud/huawei/keypair.go | 61 ++++++++------ .../x/cloudmux/pkg/multicloud/huawei/quota.go | 80 ------------------- .../pkg/multicloud/huawei/sslcertificate.go | 58 ++++++-------- 11 files changed, 74 insertions(+), 236 deletions(-) delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go diff --git a/go.mod b/go.mod index 7445a7163e7..dcb12fcb8ba 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072105-9257091d5b31 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063422-30c5f1f76541 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 diff --git a/go.sum b/go.sum index 0850812a36a..972adaeea3c 100644 --- a/go.sum +++ b/go.sum @@ -1201,8 +1201,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072105-9257091d5b31 h1:AdJrsaKEjnohmLNNr2WOUoFfG87GYRJ6JNXqnMuvTx8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072105-9257091d5b31/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063422-30c5f1f76541 h1:dmT7zzhP95ybI68F178+E41ve/M8JT9bC45DEZPXy4A= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063422-30c5f1f76541/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/compute/models/sslcertificate.go b/pkg/compute/models/sslcertificate.go index 5247a715c26..39697b82fef 100644 --- a/pkg/compute/models/sslcertificate.go +++ b/pkg/compute/models/sslcertificate.go @@ -74,8 +74,8 @@ type SSSLCertificate struct { Fingerprint string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"` City string `width:"2048" charset:"utf8" nullable:"false" list:"user" create:"required"` OrgName string `width:"2048" charset:"utf8" nullable:"false" list:"user" create:"required"` - Certificate string `charset:"utf8" nullable:"false" list:"user" create:"required"` - PrivateKey string `charset:"utf8" nullable:"false" list:"user" create:"required"` + Certificate string `charset:"utf8" nullable:"true" list:"user" create:"required"` + PrivateKey string `charset:"utf8" nullable:"true" list:"user" create:"required"` } func (s SSSLCertificate) GetExternalId() string { diff --git a/vendor/modules.txt b/vendor/modules.txt index 95a8dbb6ea1..ebb5a60d26d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1465,7 +1465,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072105-9257091d5b31 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063422-30c5f1f76541 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go index 60963fd57fa..d6dc91b9c29 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go @@ -37,7 +37,6 @@ type Client struct { OpenStackImages *modules.SImageManager Interface *modules.SInterfaceManager Jobs *modules.SJobManager - Keypairs *modules.SKeypairManager Orders *modules.SOrderManager Port *modules.SPortManager Projects *modules.SProjectManager @@ -62,7 +61,6 @@ type Client struct { DBInstanceJob *modules.SDBInstanceJobManager Traces *modules.STraceManager CloudEye *modules.SCloudEyeManager - Quotas *modules.SQuotaManager EnterpriseProjects *modules.SEnterpriseProjectManager Roles *modules.SRoleManager Groups *modules.SGroupManager @@ -123,7 +121,6 @@ func (self *Client) SetHttpClient(httpClient *http.Client) { self.DcsAvailableZone.SetHttpClient(httpClient) self.Disks.SetHttpClient(httpClient) self.Domains.SetHttpClient(httpClient) - self.Keypairs.SetHttpClient(httpClient) self.Orders.SetHttpClient(httpClient) self.Subnets.SetHttpClient(httpClient) self.Users.SetHttpClient(httpClient) @@ -193,7 +190,6 @@ func (self *Client) initManagers() { self.DcsAvailableZone = modules.NewDcsAvailableZoneManager(self.cfg) self.Disks = modules.NewDiskManager(self.cfg) self.Domains = modules.NewDomainManager(self.cfg) - self.Keypairs = modules.NewKeypairManager(self.cfg) self.Orders = modules.NewOrderManager(self.cfg) self.Subnets = modules.NewSubnetManager(self.cfg) self.Users = modules.NewUserManager(self.cfg) @@ -213,7 +209,6 @@ func (self *Client) initManagers() { self.DBInstanceJob = modules.NewDBInstanceJobManager(self.cfg) self.Traces = modules.NewTraceManager(self.cfg) self.CloudEye = modules.NewCloudEyeManager(self.cfg) - self.Quotas = modules.NewQuotaManager(self.cfg) self.EnterpriseProjects = modules.NewEnterpriseProjectManager(self.cfg) self.Roles = modules.NewRoleManager(self.cfg) self.Groups = modules.NewGroupManager(self.cfg) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go deleted file mode 100644 index b25b4e33c8e..00000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package modules - -import ( - "yunion.io/x/cloudmux/pkg/multicloud/huawei/client/manager" -) - -type SKeypairManager struct { - SResourceManager -} - -func NewKeypairManager(cfg manager.IManagerConfig) *SKeypairManager { - return &SKeypairManager{SResourceManager: SResourceManager{ - SBaseManager: NewBaseManager(cfg), - ServiceName: ServiceNameECS, - Region: cfg.GetRegionId(), - ProjectId: cfg.GetProjectId(), - version: "v2", - Keyword: "keypair", - KeywordPlural: "keypairs", - - ResourceKeyword: "os-keypairs", - }} -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go deleted file mode 100644 index e7a488a828a..00000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package modules - -import ( - "yunion.io/x/cloudmux/pkg/multicloud/huawei/client/manager" -) - -type SQuotaManager struct { - SResourceManager -} - -func NewQuotaManager(cfg manager.IManagerConfig) *SQuotaManager { - return &SQuotaManager{SResourceManager: SResourceManager{ - SBaseManager: NewBaseManager(cfg), - ServiceName: ServiceNameEVS, - Region: cfg.GetRegionId(), - ProjectId: cfg.GetProjectId(), - version: "v1", - Keyword: "quotas", - KeywordPlural: "quotas", - - ResourceKeyword: "quotas", - }} -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go index 0234c962d74..3e1f55fbf03 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go @@ -631,21 +631,9 @@ func (self *SHuaweiClient) QueryAccountBalance() (*SBalance, error) { } func (self *SHuaweiClient) GetISSLCertificates() ([]cloudprovider.ICloudSSLCertificate, error) { - ret := make([]SSSLCertificate, 0) - offset := 0 - - for { - part, total, err := self.GetSSLCertificates(50, offset) - if err != nil { - return nil, errors.Wrapf(err, "GetSSLCertificates") - } - - ret = append(ret, part...) - if len(ret) >= total { - break - } - - offset += 50 + ret, err := self.GetSSLCertificates() + if err != nil { + return nil, errors.Wrapf(err, "GetSSLCertificates") } result := make([]cloudprovider.ICloudSSLCertificate, 0) @@ -858,7 +846,7 @@ func (self *SHuaweiClient) getUrl(service, regionId, resource string, method htt case SERVICE_NAT: url = fmt.Sprintf("https://nat.%s.myhuaweicloud.com/v2/%s/%s", regionId, self.projectId, resource) case SERVICE_SCM: - url = fmt.Sprintf("https://scm.%s.myhuaweicloud.com/v3/%s", HUAWEI_DEFAULT_REGION, resource) + url = fmt.Sprintf("https://scm.cn-north-4.myhuaweicloud.com/v3/%s", resource) case SERVICE_CDN: url = fmt.Sprintf("https://cdn.myhuaweicloud.com/v1.0/%s", resource) case SERVICE_GAUSSDB, SERVICE_GAUSSDB_NOSQL: diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go index f0d4db7f148..65c551ab328 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go @@ -16,6 +16,7 @@ package huawei import ( "fmt" + "net/url" "strconv" "strings" "time" @@ -23,14 +24,15 @@ import ( "github.com/aokoli/goutils" "golang.org/x/crypto/ssh" - "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" ) -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html type SKeypair struct { - Fingerprint string `json:"fingerprint"` - Name string `json:"name"` - PublicKey string `json:"public_key"` + Keypair struct { + Fingerprint string `json:"fingerprint"` + Name string `json:"name"` + PublicKey string `json:"public_key"` + } } func (self *SRegion) getFingerprint(publicKey string) (string, error) { @@ -43,15 +45,22 @@ func (self *SRegion) getFingerprint(publicKey string) (string, error) { return fingerprint, nil } -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html -func (self *SRegion) GetKeypairs() ([]SKeypair, int, error) { - keypairs := make([]SKeypair, 0) - err := doListAll(self.ecsClient.Keypairs.List, nil, &keypairs) - return keypairs, len(keypairs), err +func (self *SRegion) GetKeypairs() ([]SKeypair, error) { + ret := []SKeypair{} + query := url.Values{} + resp, err := self.list(SERVICE_ECS, "os-keypairs", query) + if err != nil { + return nil, errors.Wrapf(err, "list os-keypairs") + } + err = resp.Unmarshal(&ret, "keypairs") + if err != nil { + return nil, err + } + return ret, nil } func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { - keypairs, _, err := self.GetKeypairs() + keypairs, err := self.GetKeypairs() if err != nil { return "", err } @@ -62,24 +71,30 @@ func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { } for _, keypair := range keypairs { - if keypair.Fingerprint == fingerprint { - return keypair.Name, nil + if keypair.Keypair.Fingerprint == fingerprint { + return keypair.Keypair.Name, nil } } return "", fmt.Errorf("keypair not found %s", err) } -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212678.html +// https://console.huaweicloud.com/apiexplorer/#/openapi/ECS/doc?api=NovaCreateKeypair func (self *SRegion) ImportKeypair(name, publicKey string) (*SKeypair, error) { - keypairObj := jsonutils.NewDict() - keypairObj.Add(jsonutils.NewString(name), "name") - keypairObj.Add(jsonutils.NewString(publicKey), "public_key") - params := jsonutils.NewDict() - params.Set("keypair", keypairObj) - ret := SKeypair{} - err := DoCreate(self.ecsClient.Keypairs.Create, params, &ret) - return &ret, err + params := map[string]interface{}{ + "name": name, + "public_key": publicKey, + } + resp, err := self.post(SERVICE_ECS, "os-keypairs", map[string]interface{}{"keypair": params}) + if err != nil { + return nil, errors.Wrapf(err, "create os-keypairs") + } + ret := &SKeypair{} + err = resp.Unmarshal(ret) + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + return ret, nil } func (self *SRegion) importKeypair(publicKey string) (string, error) { @@ -92,7 +107,7 @@ func (self *SRegion) importKeypair(publicKey string) (string, error) { if k, e := self.ImportKeypair(name, publicKey); e != nil { return "", fmt.Errorf("keypair import error %s", e) } else { - return k.Name, nil + return k.Keypair.Name, nil } } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go deleted file mode 100644 index a26fc137b72..00000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package huawei - -import ( - "yunion.io/x/pkg/errors" - - "yunion.io/x/cloudmux/pkg/cloudprovider" -) - -type SQuota struct { - Min int - Quota int - Type string - Used int -} - -func (q *SQuota) GetGlobalId() string { - return q.Type -} - -func (q *SQuota) GetQuotaType() string { - return q.Type -} - -func (q *SQuota) GetName() string { - return q.Type -} - -func (q *SQuota) GetDesc() string { - return "" -} - -func (q *SQuota) GetMaxQuotaCount() int { - return q.Quota -} - -func (q *SQuota) GetCurrentQuotaUsedCount() int { - return q.Used -} - -func (self *SRegion) GetQuotas() ([]SQuota, error) { - quotas := []SQuota{} - params := map[string]string{} - result, err := self.ecsClient.Quotas.Get("", params) - if err != nil { - return nil, errors.Wrap(err, "Quotas.List") - } - - err = result.Unmarshal("as, "resources") - if err != nil { - return nil, errors.Wrap(err, "result.Unmarshal") - } - - return quotas, nil -} - -func (region *SRegion) GetICloudQuotas() ([]cloudprovider.ICloudQuota, error) { - quotas, err := region.GetQuotas() - if err != nil { - return nil, errors.Wrap(err, "GetQuotas") - } - ret := []cloudprovider.ICloudQuota{} - for i := range quotas { - ret = append(ret, "as[i]) - } - return ret, nil -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go index f15bf49ec16..297a03b0e80 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go @@ -137,48 +137,42 @@ func (s *SSSLCertificate) GetKey() string { func (s *SSSLCertificate) GetDetails() (*SSSLCertificate, error) { if !s.detailsInitd { - cert, err := s.client.GetISSLCertificate(s.GetId()) + cert, err := s.client.GetSSLCertificate(s.GetId()) if err != nil { return nil, err } s.detailsInitd = true - _cert, ok := cert.(*SSSLCertificate) - if !ok { - return nil, errors.Wrapf(err, "cert.(*SSSLCertificate)") - } - s.Certificate = _cert.Certificate - s.PrivateKey = _cert.PrivateKey + s.Certificate = cert.Certificate + s.PrivateKey = cert.PrivateKey } return s, nil } -func (r *SHuaweiClient) GetSSLCertificates(size, offset int) ([]SSSLCertificate, int, error) { - if size < 1 || size > 50 { - size = 50 - } - if offset < 0 { - offset = 0 - } - - params := url.Values{ - "limit": []string{fmt.Sprintf("%d", size)}, - "offset": []string{fmt.Sprintf("%d", offset)}, - "sort_key": []string{"certExpiredTime"}, - "sort_dir": []string{"DESC"}, - } - resp, err := r.list(SERVICE_SCM, "", "scm/certificates", params) - if err != nil { - return nil, 0, errors.Wrapf(err, "CertificateList") - } - +func (r *SHuaweiClient) GetSSLCertificates() ([]SSSLCertificate, error) { + params := url.Values{} + params.Set("sort_key", "certExpiredTime") + params.Set("sort_dir", "DESC") ret := make([]SSSLCertificate, 0) - err = resp.Unmarshal(&ret, "certificates") - if err != nil { - return nil, 0, errors.Wrapf(err, "resp.Unmarshal") + for { + resp, err := r.list(SERVICE_SCM, "", "scm/certificates", params) + if err != nil { + return nil, errors.Wrapf(err, "list certificates") + } + part := struct { + Certificates []SSSLCertificate + TotalCount int + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, err + } + ret = append(ret, part.Certificates...) + if len(ret) >= part.TotalCount || len(part.Certificates) == 0 { + break + } + params.Set("offset", fmt.Sprintf("%d", len(ret))) } - - totalCount, _ := resp.Int("total_count") - return ret, int(totalCount), nil + return ret, nil } func (r *SHuaweiClient) GetSSLCertificate(certId string) (*SSSLCertificate, error) {