Skip to content

Commit

Permalink
fix(region): huawei subaccount
Browse files Browse the repository at this point in the history
  • Loading branch information
ioito committed Dec 21, 2023
1 parent 5a02dfc commit d8d0860
Show file tree
Hide file tree
Showing 37 changed files with 478 additions and 519 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -270,3 +270,5 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.0.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)

replace yunion.io/x/cloudmux => /Users/quxuan/.go/src/yunion.io/x/cloudmux
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1201,8 +1201,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231220111515-f4d6c15e4077 h1:uzfyl/PcxvNAYHwf+uOXnNRxucRIoLC3xLbdLhp68o0=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231220111515-f4d6c15e4077/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
Expand Down
6 changes: 0 additions & 6 deletions pkg/apis/compute/cloudaccount_const.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@ const (
CLOUD_ACCESS_ENV_AZURE_GERMAN = compute.CLOUD_ACCESS_ENV_AZURE_GERMAN
CLOUD_ACCESS_ENV_AZURE_US_GOVERNMENT = compute.CLOUD_ACCESS_ENV_AZURE_US_GOVERNMENT
CLOUD_ACCESS_ENV_AZURE_CHINA = compute.CLOUD_ACCESS_ENV_AZURE_CHINA
CLOUD_ACCESS_ENV_HUAWEI_GLOBAL = compute.CLOUD_ACCESS_ENV_HUAWEI_GLOBAL
CLOUD_ACCESS_ENV_HUAWEI_CHINA = compute.CLOUD_ACCESS_ENV_HUAWEI_CHINA
CLOUD_ACCESS_ENV_ALIYUN_GLOBAL = compute.CLOUD_ACCESS_ENV_ALIYUN_GLOBAL
CLOUD_ACCESS_ENV_ALIYUN_FINANCE = compute.CLOUD_ACCESS_ENV_ALIYUN_FINANCE
CLOUD_ACCESS_ENV_CTYUN_CHINA = compute.CLOUD_ACCESS_ENV_CTYUN_CHINA
Expand Down Expand Up @@ -287,10 +285,6 @@ var (
"InternationalCloud": CLOUD_ACCESS_ENV_AWS_GLOBAL,
"ChinaCloud": CLOUD_ACCESS_ENV_AWS_CHINA,
},
CLOUD_PROVIDER_HUAWEI: {
"InternationalCloud": CLOUD_ACCESS_ENV_HUAWEI_GLOBAL,
"ChinaCloud": CLOUD_ACCESS_ENV_HUAWEI_CHINA,
},
CLOUD_PROVIDER_ALIYUN: {
"InternationalCloud": CLOUD_PROVIDER_ALIYUN,
"FinanceCloud": CLOUD_ACCESS_ENV_ALIYUN_FINANCE,
Expand Down
3 changes: 1 addition & 2 deletions pkg/compute/models/cloudimages.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package models
import (
"context"
"database/sql"
"fmt"

"yunion.io/x/log"
"yunion.io/x/pkg/errors"
Expand Down Expand Up @@ -150,7 +149,7 @@ func (self *SCloudimage) syncWithImage(ctx context.Context, userCred mcclient.To
return err
}

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.ImageBase, region.ExternalId, image.GetGlobalId())
skuUrl := region.getMetaUrl(meta.ImageBase, image.GetGlobalId())

obj, err := db.FetchByExternalId(CachedimageManager, image.GetGlobalId())
if err != nil {
Expand Down
3 changes: 1 addition & 2 deletions pkg/compute/models/cloudregions.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package models
import (
"context"
"database/sql"
"fmt"
"strings"
"time"

Expand Down Expand Up @@ -1275,7 +1274,7 @@ func (self *SCloudregion) newCloudimage(ctx context.Context, userCred mcclient.T
return err
}

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.ImageBase, self.ExternalId, iImage.GetGlobalId())
skuUrl := self.getMetaUrl(meta.ImageBase, iImage.GetGlobalId())
err = meta.Get(skuUrl, image)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down
2 changes: 1 addition & 1 deletion pkg/compute/models/dbinstance_skus.go
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ func (self *SCloudregion) newDBInstanceSkuFromCloudSku(ctx context.Context, user
sku := &SDBInstanceSku{}
sku.SetModelManager(DBInstanceSkuManager, sku)

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.DBInstanceBase, self.ExternalId, externalId)
skuUrl := self.getMetaUrl(meta.DBInstanceBase, externalId)
err = meta.Get(skuUrl, sku)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down
2 changes: 1 addition & 1 deletion pkg/compute/models/elasticcache_skus.go
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ func (self *SCloudregion) newFromPublicCloudSku(ctx context.Context, userCred mc
zoneMaps[zone.ExternalId] = zone.Id
}

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.ElasticCacheBase, self.ExternalId, externalId)
skuUrl := self.getMetaUrl(meta.ElasticCacheBase, externalId)
sku := &SElasticcacheSku{}
sku.SetModelManager(ElasticcacheSkuManager, sku)
err = meta.Get(skuUrl, sku)
Expand Down
2 changes: 1 addition & 1 deletion pkg/compute/models/nas_skus.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ func (self *SCloudregion) newFromCloudNasSku(ctx context.Context, userCred mccli
sku := &SNasSku{}
sku.SetModelManager(NasSkuManager, sku)

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.NasBase, self.ExternalId, isku.GetGlobalId())
skuUrl := self.getMetaUrl(meta.NasBase, isku.GetGlobalId())
err = meta.Get(skuUrl, sku)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down
2 changes: 1 addition & 1 deletion pkg/compute/models/nat_skus.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ func (self *SCloudregion) newFromCloudNatSku(ctx context.Context, userCred mccli
sku := &SNatSku{}
sku.SetModelManager(NatSkuManager, sku)

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.NatBase, self.ExternalId, isku.GetGlobalId())
skuUrl := self.getMetaUrl(meta.NatBase, isku.GetGlobalId())
err = meta.Get(skuUrl, sku)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down
12 changes: 10 additions & 2 deletions pkg/compute/models/server_skus.go
Original file line number Diff line number Diff line change
Expand Up @@ -1162,6 +1162,14 @@ func (self *SServerSku) setPrepaidPostpaidStatus(userCred mcclient.TokenCredenti
return nil
}

func (region *SCloudregion) getMetaUrl(base string, externalId string) string {
if region.Provider == api.CLOUD_PROVIDER_HUAWEI && strings.Contains(region.ExternalId, "_") {
idx := strings.Index(region.ExternalId, "_")
return fmt.Sprintf("%s/%s/%s.json", base, region.ExternalId[:idx], externalId)
}
return fmt.Sprintf("%s/%s/%s.json", base, region.ExternalId, externalId)
}

func (region *SCloudregion) newPublicCloudSku(ctx context.Context, userCred mcclient.TokenCredential, extSku SServerSku) error {
meta, err := yunionmeta.FetchYunionmeta(ctx)
if err != nil {
Expand All @@ -1179,7 +1187,7 @@ func (region *SCloudregion) newPublicCloudSku(ctx context.Context, userCred mccl
sku := &SServerSku{}
sku.SetModelManager(ServerSkuManager, sku)

skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.ServerBase, region.ExternalId, extSku.ExternalId)
skuUrl := region.getMetaUrl(meta.ServerBase, extSku.ExternalId)
err = meta.Get(skuUrl, sku)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down Expand Up @@ -1227,7 +1235,7 @@ func (self *SServerSku) syncWithCloudSku(ctx context.Context, userCred mcclient.
}

sku := &SServerSku{}
skuUrl := fmt.Sprintf("%s/%s/%s.json", meta.ServerBase, region.ExternalId, extSku.ExternalId)
skuUrl := region.getMetaUrl(meta.ServerBase, extSku.ExternalId)
err = meta.Get(skuUrl, sku)
if err != nil {
return errors.Wrapf(err, "Get")
Expand Down
2 changes: 1 addition & 1 deletion pkg/mcclient/options/cloudaccounts.go
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ func (opts *SOpenStackCloudAccountCreateOptions) Params() (jsonutils.JSONObject,

type SHuaweiCloudAccountCreateOptions struct {
SCloudAccountCreateBaseOptions
SAccessKeyCredentialWithEnvironment
SAccessKeyCredential
}

func (opts *SHuaweiCloudAccountCreateOptions) Params() (jsonutils.JSONObject, error) {
Expand Down
7 changes: 6 additions & 1 deletion pkg/util/yunionmeta/meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ import (
"yunion.io/x/pkg/util/httputils"
"yunion.io/x/pkg/util/version"

"yunion.io/x/onecloud/pkg/esxi/options"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
)
Expand Down Expand Up @@ -152,6 +153,10 @@ func (self *SSkuResourcesMeta) Index(resType string) (map[string]string, error)
}

func (self *SSkuResourcesMeta) List(resType string, regionId string, retVal interface{}) error {
if strings.HasPrefix(regionId, api.CLOUD_PROVIDER_HUAWEI) && strings.Contains(regionId, "_") {
idx := strings.Index(regionId, "_")
regionId = regionId[:idx]
}
var url string
switch resType {
case "dbinstance_sku":
Expand Down
168 changes: 168 additions & 0 deletions scripts/upgrade_3_11.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#!/usr/bin/env bash

# 登录cloudpods控制节点获取mysql账号密码信息,填写到下面变量
# kubectl get oc -n onecloud default -o yaml | grep -A 4 mysql
#
# 升级到3.11版本之后,等待所有华为云账号同步完成后,执行此升级脚本
# 脚本原理如下:
# 脚本会检索1天内deleted=1的华为云资源,通过external_id反查当前数据库中对应external_id相同且deleted=0的资源
# 先会将deleted=1的资源id设为id=id-old, 再将deleted=0的资源id变更到deleted=1的资源id
#
HOST='127.0.0.1'
USERNAME='root'
PASSWORD=''
DATETIME=''
if ["$(uname)"=="Darwin"]; then
DATETIME=$(date -v-1d -u "+%Y-%m-%dT%H:%M%SZ")
else
DATETIME=$(date -d "1 day ago" -u "+%Y-%m-%dT%H:%M%SZ")
if

function exec_sql_with_db() {
export MYSQL_PWD=$PASSWORD
local db=$1
local sql=$2
echo $(mysql -u"$USERNAME" -h "$HOST" -D"$db" -s -e "$sql")
}

function get_relation_tables() {
local field=$1
exec_sql_with_db "information_schema" "select table_name from columns where column_name='$field' and table_schema='yunioncloud'"
}

function exec_sql() {
exec_sql_with_db "yunioncloud" "$1"
}

function uuid() {
echo $1 | awk -F '-' '{print $1"-"$2"-"$3"-"$4"-"$5}'
}

function change_uuid() {
local table=$1
local target_id=$2

uid=$(uuid $target_id)
if [ "$uid" != "$target_id" ]; then
target_id=$uid
else
exec_sql "update $table set id='$target_id-old' where id='$target_id'"
fi
echo $target_id
}


echo "upgrade guests_tbl"
exec_sql "select name, id, external_id from guests_tbl where deleted=1 and length(external_id) > 0 and host_id in (select id from hosts_tbl where deleted=1 and manager_id in (select id from cloudproviders_tbl where deleted=1 and provider='Huawei' and deleted_at > '$DATETIME'))" | while read -r line; do
info=($(echo $line | tr " ", "\n"))
name=${info[0]}
target_id=${info[1]}
external_id=${info[2]}
id=$(exec_sql "select id from guests_tbl where deleted=0 and external_id='$external_id' and host_id in (select id from hosts_tbl where deleted=0 and manager_id in (select id from cloudproviders_tbl where deleted=0 and provider='Huawei'))")
if [ -n "$id" ]; then
target_id=$(change_uuid "guests_tbl" $target_id)
echo "change server $name id from $id => $target_id"

exec_sql "update guests_tbl set id='$target_id' where id='$id' and deleted=0"

tables=$(get_relation_tables "guest_id")
for table in $tables; do
exec_sql "update $table set guest_id='$target_id' where guest_id='$id' and deleted=0"
done

fi

done

echo "upgrade disks_tbl"
exec_sql "select name, id, external_id from disks_tbl where deleted=1 and length(external_id) > 0 and storage_id in (select id from storages_tbl where deleted=1 and manager_id in (select id from cloudproviders_tbl where deleted=1 and provider='Huawei' and deleted_at > '$DATETIME'))" | while read -r line; do
info=($(echo $line | tr " ", "\n"))
name=${info[0]}
target_id=${info[1]}
external_id=${info[2]}
id=$(exec_sql "select id from disks_tbl where deleted=0 and external_id='$external_id' and storage_id in (select id from storages_tbl where deleted=0 and manager_id in (select id from cloudproviders_tbl where deleted=0 and provider='Huawei'))")
if [ -n "$id" ]; then
target_id=$(change_uuid "disks_tbl" $target_id)
echo "change disk $name id from $id => $target_id"

exec_sql "update disks_tbl set id='$target_id' where id='$id' and deleted=0"

tables=$(get_relation_tables "disk_id")
for table in $tables; do
exec_sql "update $table set disk_id='$target_id' where disk_id='$id' and deleted=0"
done

fi

done



echo "upgrade networks_tbl"
exec_sql "select name, id, external_id from networks_tbl where deleted=1 and length(external_id) > 0 and wire_id in (select id from wires_tbl where deleted=1 and vpc_id in (select id from vpcs_tbl where deleted=1 and manager_id in (select id from cloudproviders_tbl where deleted=1 and provider='Huawei' and deleted_at > '$DATETIME')))" | while read -r line; do
info=($(echo $line | tr " ", "\n"))
name=${info[0]}
target_id=${info[1]}
external_id=${info[2]}
id=$(exec_sql "select id from networks_tbl where deleted=0 and external_id='$external_id' and wire_id in (select id from wires_tbl where deleted=0 and vpc_id in (select id from vpcs_tbl where deleted=0 and manager_id in (select id from cloudproviders_tbl where deleted=0 and provider='Huawei')))")
if [ -n "$id" ]; then
target_id=$(change_uuid "networks_tbl" $target_id)
echo "change network $name id from $id => $target_id"

exec_sql "update networks_tbl set id='$target_id' where id='$id' and deleted=0"

tables=$(get_relation_tables "network_id")
for table in $tables; do
if [ "$table" == "network_additional_wire_tbl" ]; then
exec_sql "update $table set network_id='$target_id' where network_id='$id'"
else
exec_sql "update $table set network_id='$target_id' where network_id='$id' and deleted=0"
fi
done

fi
done


function upgrade_managed_resources_table() {
local res_type=$1
local table_name=$2
local field=$3

echo "upgrade $table_name"
exec_sql "select name, id, external_id from $table_name where deleted=1 and length(external_id) > 0 and manager_id in (select id from cloudproviders_tbl where deleted=1 and provider='Huawei' and deleted_at > '$DATETIME')" | while read -r line; do
info=($(echo $line | tr " ", "\n"))
name=${info[0]}
target_id=${info[1]}
external_id=${info[2]}
id=$(exec_sql "select id from $table_name where deleted=0 and external_id='$external_id' and manager_id in (select id from cloudproviders_tbl where deleted=0 and provider='Huawei')")
if [ -n "$id" ]; then
target_id=$(change_uuid $table_name $target_id)
echo "change $res_type $name id from $id => $target_id"

exec_sql "update $table_name set id='$target_id' where id='$id' and deleted=0"

tables=$(get_relation_tables $field)
for table in $tables; do
exec_sql "update $table set $field='$target_id' where $field='$id' and deleted=0"
done
fi
done
}


upgrade_managed_resources_table "elasticcacheinstance" "elasticcacheinstances_tbl" "elasticcache_id"
upgrade_managed_resources_table "dbinstance" "dbinstances_tbl" "dbinstance_id"
upgrade_managed_resources_table "bucket" "buckets_tbl" "bucket_id"
upgrade_managed_resources_table "vpc" "vpcs_tbl" "vpc_id"
upgrade_managed_resources_table "cdn" "cdn_domains_tbl" "cdn_domain_id"
upgrade_managed_resources_table "dns" "dnszones_tbl" "dns_zone_id"
upgrade_managed_resources_table "elastic_search" "elastic_searchs_tbl" "elastic_search_id"
upgrade_managed_resources_table "eip" "elasticips_tbl" "eip_id"
upgrade_managed_resources_table "inter_vpc_network" "inter_vpc_networks_tbl" "inter_vpc_network_id"
upgrade_managed_resources_table "kafka" "kafkas_tbl" "kafka_id"
upgrade_managed_resources_table "loadbalancer" "loadbalancers_tbl" "loadbalancer_id"
upgrade_managed_resources_table "snapshot" "snapshots_tbl" "snapshot_id"
upgrade_managed_resources_table "sslcertificate" "sslcertificates_tbl" "sslcertificate_id"


3 changes: 2 additions & 1 deletion vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1465,7 +1465,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231220111515-f4d6c15e4077
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231220111515-f4d6c15e4077 => /Users/quxuan/.go/src/yunion.io/x/cloudmux
## explicit; go 1.18
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
Expand Down Expand Up @@ -1625,3 +1625,4 @@ yunion.io/x/sqlchemy/backends/sqlite
# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
## explicit; go 1.12
yunion.io/x/structarg
# yunion.io/x/cloudmux => /Users/quxuan/.go/src/yunion.io/x/cloudmux

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 6 additions & 5 deletions vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d8d0860

Please sign in to comment.