From d478a8ac8d497eb944b3b6a8aa508ae892f4c396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 15 Dec 2023 17:07:56 +0800 Subject: [PATCH] fix(region): refactor snapshot policy (#18996) --- cmd/climc/shell/compute/snapshot_policy.go | 112 +- .../shell/compute/snapshotpolicy_disk.go | 80 -- go.mod | 2 +- go.sum | 4 +- pkg/apis/compute/disk.go | 4 + pkg/apis/compute/input.go | 5 +- pkg/apis/compute/snapshot_const.go | 26 +- pkg/apis/compute/snapshotpolicy.go | 186 +-- pkg/apis/compute/snapshotpolicycache.go | 25 - pkg/apis/compute/snapshotpolicydisk.go | 36 - pkg/apis/compute/zz_generated.model.go | 3 +- pkg/compute/models/capabilities.go | 9 + pkg/compute/models/cloudsync.go | 11 +- pkg/compute/models/disks.go | 336 +++-- pkg/compute/models/purge.go | 4 +- pkg/compute/models/regiondrivers.go | 12 +- pkg/compute/models/snapshotpolicy.go | 1132 +++++++---------- pkg/compute/models/snapshotpolicy_test.go | 208 --- pkg/compute/models/snapshotpolicycache.go | 566 --------- pkg/compute/models/snapshotpolicydisks.go | 615 +-------- pkg/compute/models/snapshotpolicyresource.go | 180 --- pkg/compute/models/snapshots.go | 5 +- pkg/compute/options/options.go | 5 +- pkg/compute/regiondrivers/aliyun.go | 18 - pkg/compute/regiondrivers/base.go | 44 +- pkg/compute/regiondrivers/kvm.go | 98 +- pkg/compute/regiondrivers/managedvirtual.go | 230 ++-- pkg/compute/regiondrivers/qcloud.go | 28 - pkg/compute/service/handlers.go | 4 +- .../tasks/disk_clean_overdued_snapshots.go | 11 +- pkg/compute/tasks/disk_delete_task.go | 3 +- .../snapshot_policy_cache_delete_task.go | 64 - .../tasks/snapshotpolicy_bind_disks_task.go | 72 ++ .../tasks/snapshotpolicy_create_task.go | 73 ++ ..._task.go => snapshotpolicy_delete_task.go} | 30 +- pkg/compute/tasks/snapshotpolicy_disk_task.go | 242 ---- .../tasks/snapshotpolicy_syncstatus_task.go | 62 + .../tasks/snapshotpolicy_unbind_disks_task.go | 72 ++ .../compute/mod_snapshotpolicycache.go | 31 - .../modules/compute/mod_snapshotpolicydisk.go | 45 - .../options/compute/snapshotpolicy.go | 54 + pkg/util/logclient/consts.go | 1 + vendor/modules.txt | 2 +- .../pkg/apis/compute/snapshot_const.go | 5 - vendor/yunion.io/x/cloudmux/pkg/apis/const.go | 1 + .../x/cloudmux/pkg/cloudprovider/consts.go | 1 + .../x/cloudmux/pkg/cloudprovider/resources.go | 13 +- .../pkg/cloudprovider/snapshot_policy.go | 9 +- .../cloudmux/pkg/multicloud/aliyun/aliyun.go | 1 + .../x/cloudmux/pkg/multicloud/aliyun/disk.go | 65 +- .../pkg/multicloud/aliyun/instance.go | 13 +- .../pkg/multicloud/aliyun/snapshot_policy.go | 176 ++- .../cloudmux/pkg/multicloud/aliyun/storage.go | 46 +- .../cloudmux/pkg/multicloud/apsara/apsara.go | 1 + .../x/cloudmux/pkg/multicloud/apsara/disk.go | 62 +- .../pkg/multicloud/apsara/instance.go | 12 +- .../pkg/multicloud/apsara/snapshot_policy.go | 78 +- .../cloudmux/pkg/multicloud/apsara/storage.go | 14 +- .../pkg/multicloud/bingocloud/disk.go | 4 - .../x/cloudmux/pkg/multicloud/ctyun/disk.go | 4 - .../x/cloudmux/pkg/multicloud/ctyun/region.go | 28 - .../x/cloudmux/pkg/multicloud/disk_base.go | 4 - .../x/cloudmux/pkg/multicloud/ecloud/disk.go | 4 - .../x/cloudmux/pkg/multicloud/google/disk.go | 9 - .../cloudmux/pkg/multicloud/google/google.go | 1 + .../pkg/multicloud/google/resourcepolicy.go | 29 +- .../x/cloudmux/pkg/multicloud/jdcloud/disk.go | 4 - .../pkg/multicloud/objectstore/objectstore.go | 4 - .../pkg/multicloud/oracle/boot-disk.go | 4 - .../x/cloudmux/pkg/multicloud/oracle/disk.go | 4 - .../x/cloudmux/pkg/multicloud/qcloud/disk.go | 4 - .../cloudmux/pkg/multicloud/qcloud/qcloud.go | 1 + .../pkg/multicloud/qcloud/snapshot_policy.go | 59 +- .../x/cloudmux/pkg/multicloud/region_base.go | 18 +- .../pkg/multicloud/remotefile/disk.go | 4 - .../pkg/multicloud/volcengine/disk.go | 4 - 76 files changed, 1628 insertions(+), 3808 deletions(-) delete mode 100644 cmd/climc/shell/compute/snapshotpolicy_disk.go delete mode 100644 pkg/apis/compute/snapshotpolicycache.go delete mode 100644 pkg/apis/compute/snapshotpolicydisk.go delete mode 100644 pkg/compute/models/snapshotpolicy_test.go delete mode 100644 pkg/compute/models/snapshotpolicycache.go delete mode 100644 pkg/compute/models/snapshotpolicyresource.go delete mode 100644 pkg/compute/tasks/snapshot_policy_cache_delete_task.go create mode 100644 pkg/compute/tasks/snapshotpolicy_bind_disks_task.go create mode 100644 pkg/compute/tasks/snapshotpolicy_create_task.go rename pkg/compute/tasks/{snapshot_policy_delete_task.go => snapshotpolicy_delete_task.go} (70%) delete mode 100644 pkg/compute/tasks/snapshotpolicy_disk_task.go create mode 100644 pkg/compute/tasks/snapshotpolicy_syncstatus_task.go create mode 100644 pkg/compute/tasks/snapshotpolicy_unbind_disks_task.go delete mode 100644 pkg/mcclient/modules/compute/mod_snapshotpolicycache.go delete mode 100644 pkg/mcclient/modules/compute/mod_snapshotpolicydisk.go create mode 100644 pkg/mcclient/options/compute/snapshotpolicy.go diff --git a/cmd/climc/shell/compute/snapshot_policy.go b/cmd/climc/shell/compute/snapshot_policy.go index 103307f82b1..ff4744f407c 100644 --- a/cmd/climc/shell/compute/snapshot_policy.go +++ b/cmd/climc/shell/compute/snapshot_policy.go @@ -15,112 +15,18 @@ package compute import ( - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/cmd/climc/shell" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/mcclient/options/compute" ) func init() { - type SnapshotPolicyListOptions struct { - options.BaseListOptions - - OrderByBindDiskCount string - } - R(&SnapshotPolicyListOptions{}, "snapshot-policy-list", "List snapshot policy", func(s *mcclient.ClientSession, args *SnapshotPolicyListOptions) error { - params, err := options.ListStructToParams(args) - if err != nil { - return err - } - result, err := modules.SnapshotPoliciy.List(s, params) - if err != nil { - return err - } - printList(result, modules.SnapshotPoliciy.GetColumns(s)) - return nil - }) - - type SnapshotPolicyDeleteOptions struct { - ID string `help:"Delete snapshot id"` - } - R(&SnapshotPolicyDeleteOptions{}, "snapshot-policy-delete", "Delete snapshot policy", func(s *mcclient.ClientSession, args *SnapshotPolicyDeleteOptions) error { - result, err := modules.SnapshotPoliciy.Delete(s, args.ID, nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type SnapshotPolicyCreateOptions struct { - NAME string - - RetentionDays int `help:"snapshot retention days"` - RepeatWeekdays []int `help:"snapshot create days on week"` - TimePoints []int `help:"snapshot create time points on one day"` - } - - R(&SnapshotPolicyCreateOptions{}, "snapshot-policy-create", "Create snapshot policy", func(s *mcclient.ClientSession, args *SnapshotPolicyCreateOptions) error { - params := jsonutils.Marshal(args).(*jsonutils.JSONDict) - snapshot, err := modules.SnapshotPoliciy.Create(s, params) - if err != nil { - return err - } - printObject(snapshot) - return nil - }) - - type SnapshotPolicyBindDisksOptions struct { - ID string `help:"ID"` - Disk []string `help:"ids of disk"` - } - - R(&SnapshotPolicyBindDisksOptions{}, "snapshot-policy-bind-disk", "bind snapshotpolicy to disks", - func(s *mcclient.ClientSession, opts *SnapshotPolicyBindDisksOptions) error { - params, err := options.StructToParams(opts) - if err != nil { - return err - } - sp, err := modules.SnapshotPoliciy.PerformAction(s, opts.ID, "bind-disks", params) - if err != nil { - return err - } - printObject(sp) - return nil - }) - - R(&SnapshotPolicyBindDisksOptions{}, "snapshot-policy-unbind-disk", "bind snapshotpolicy to disks", - func(s *mcclient.ClientSession, opts *SnapshotPolicyBindDisksOptions) error { - params, err := options.StructToParams(opts) - if err != nil { - return err - } - sp, err := modules.SnapshotPoliciy.PerformAction(s, opts.ID, "unbind-disks", params) - if err != nil { - return err - } - printObject(sp) - return nil - }) - - type SnapshotPolicyCacheOptions struct { - ID string `help:"SnasphotPolicy ID"` - REGIONID string `help:"Region ID"` - PROVIDER string `help:"Provider ID"` - } - R(&SnapshotPolicyCacheOptions{}, "snapshot-policy-cache", "upload local snapshotpolicy to cloud", - func(s *mcclient.ClientSession, opts *SnapshotPolicyCacheOptions) error { - params, err := options.StructToParams(opts) - if err != nil { - return err - } - sp, err := modules.SnapshotPoliciy.PerformAction(s, opts.ID, "cache", params) - if err != nil { - return err - } - printObject(sp) - return nil - }, - ) + cmd := shell.NewResourceCmd(&modules.SnapshotPoliciy).WithKeyword("snapshot-policy") + cmd.List(&compute.SnapshotPolicyListOptions{}) + cmd.Delete(&options.BaseIdOptions{}) + cmd.Create(&compute.SnapshotPolicyCreateOptions{}) + cmd.Perform("bind-disks", &compute.SnapshotPolicyDisksOptions{}) + cmd.Perform("unbind-disks", &compute.SnapshotPolicyDisksOptions{}) + cmd.Perform("syncstatus", &options.BaseIdOptions{}) } diff --git a/cmd/climc/shell/compute/snapshotpolicy_disk.go b/cmd/climc/shell/compute/snapshotpolicy_disk.go deleted file mode 100644 index bac054c8586..00000000000 --- a/cmd/climc/shell/compute/snapshotpolicy_disk.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import ( - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/mcclient" - modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" -) - -func init() { - type SSnapshotpolicyDiskListOfDiskOption struct { - DISKID string `help:"disk id"` - } - R(&SSnapshotpolicyDiskListOfDiskOption{}, "disk-snapshot-policy-list", "List snapshot policy attached", - func(s *mcclient.ClientSession, opts *SSnapshotpolicyDiskListOfDiskOption) error { - result, err := modules.SnapshotPolicyDisk.ListDescendent(s, opts.DISKID, jsonutils.NewDict()) - if err != nil { - return err - } - printList(result, modules.SnapshotPolicyDisk.GetColumns(s)) - return nil - }, - ) - - type SSnapshotpolicyDiskListOfSnapshotpolicyOption struct { - SNAPSHOTPOLICYID string `help:"snapshot policy id"` - } - R(&SSnapshotpolicyDiskListOfSnapshotpolicyOption{}, "snapshot-policy-disk-list", "List disk attached", - func(s *mcclient.ClientSession, opts *SSnapshotpolicyDiskListOfSnapshotpolicyOption) error { - result, err := modules.SnapshotPolicyDisk.ListDescendent2(s, opts.SNAPSHOTPOLICYID, jsonutils.NewDict()) - if err != nil { - return err - } - printList(result, modules.SnapshotPolicyDisk.GetColumns(s)) - return nil - }, - ) - - type SSnapshotpolicyDiskAttachOption struct { - DISKID string `help:"disk id"` - SNAPSHOTPOLICYID string `help:"snapshot policy id"` - } - R(&SSnapshotpolicyDiskAttachOption{}, "disk-snapshot-policy-attach", "attach snapshot policy to disk", - func(s *mcclient.ClientSession, opts *SSnapshotpolicyDiskAttachOption) error { - _, err := modules.SnapshotPolicyDisk.Attach(s, opts.DISKID, opts.SNAPSHOTPOLICYID, jsonutils.NewDict()) - if err != nil { - return err - } - return nil - }, - ) - - type SSnapshotpolicyDiskDetachOption struct { - DISKID string `help:"disk id"` - SNAPSHOTPOLICYID string `help:"snapshot policy id"` - } - R(&SSnapshotpolicyDiskDetachOption{}, "disk-snapshot-policy-detach", "detach snapshot policy to disk", - func(s *mcclient.ClientSession, opts *SSnapshotpolicyDiskDetachOption) error { - _, err := modules.SnapshotPolicyDisk.Detach(s, opts.DISKID, opts.SNAPSHOTPOLICYID, jsonutils.NewDict()) - if err != nil { - return err - } - return nil - }, - ) -} diff --git a/go.mod b/go.mod index d63496e3839..2b1cfd11c9d 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231214030208-02cfe93ac519 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231215084755-601e4d02e207 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 diff --git a/go.sum b/go.sum index 6aaac2cc2ed..6f3d2317902 100644 --- a/go.sum +++ b/go.sum @@ -1201,8 +1201,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231214030208-02cfe93ac519 h1:0LEcNL0GkI2h/Qy2st4Dc6I9nY1MgrHSD0e7Irc/8ik= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231214030208-02cfe93ac519/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231215084755-601e4d02e207 h1:p2R9I3/DNwTJrD/RDQlF9a8xWYo4nXOs+B6KLZXqNSc= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231215084755-601e4d02e207/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/apis/compute/disk.go b/pkg/apis/compute/disk.go index 98534ff78cc..8f89a28981b 100644 --- a/pkg/apis/compute/disk.go +++ b/pkg/apis/compute/disk.go @@ -325,3 +325,7 @@ type DiskResetInput struct { SnapshotId string `json:"snapshot_id"` AutoStart bool `json:"auto_start"` } + +type DiskSnapshotpolicyInput struct { + SnapshotpolicyId string `json:"snapshotpolicy_id"` +} diff --git a/pkg/apis/compute/input.go b/pkg/apis/compute/input.go index 44c91af55e0..155a7fba62d 100644 --- a/pkg/apis/compute/input.go +++ b/pkg/apis/compute/input.go @@ -158,12 +158,13 @@ type ServiceCatalogListInput struct { type SnapshotPolicyListInput struct { apis.VirtualResourceListInput + apis.ExternalizedResourceBaseListInput + ManagedResourceListInput + RegionalFilterListInput // 按绑定的磁盘数量排序 // pattern:asc|desc OrderByBindDiskCount string `json:"order_by_bind_disk_count"` - // 是否启用？ - IsActivated *bool `json:"is_activated"` } type HostnameInput struct { diff --git a/pkg/apis/compute/snapshot_const.go b/pkg/apis/compute/snapshot_const.go index d9a08e34a31..467f7682040 100644 --- a/pkg/apis/compute/snapshot_const.go +++ b/pkg/apis/compute/snapshot_const.go @@ -14,7 +14,9 @@ package compute -import "yunion.io/x/cloudmux/pkg/apis/compute" +import ( + "yunion.io/x/cloudmux/pkg/apis/compute" +) const ( // create by @@ -29,24 +31,6 @@ const ( SNAPSHOT_DELETING = compute.SNAPSHOT_DELETING SNAPSHOT_UNKNOWN = compute.SNAPSHOT_UNKNOWN - SNAPSHOT_POLICY_CREATING = compute.SNAPSHOT_POLICY_CREATING - - SNAPSHOT_POLICY_READY = compute.SNAPSHOT_POLICY_READY - SNAPSHOT_POLICY_UPDATING = "updating" - SNAPSHOT_POLICY_UNKNOWN = compute.SNAPSHOT_POLICY_UNKNOWN - SNAPSHOT_POLICY_DELETING = "deleting" - SNAPSHOT_POLICY_DELETE_FAILED = "delete_failed" - - SNAPSHOT_POLICY_APPLY = "applying" - SNAPSHOT_POLICY_APPLY_FAILED = "apply_failed" - SNAPSHOT_POLICY_CANCEL = "canceling" - SNAPSHOT_POLICY_CANCEL_FAILED = "cancel_failed" - - SNAPSHOT_POLICY_DISK_INIT = "init" - SNAPSHOT_POLICY_DISK_READY = "ready" - SNAPSHOT_POLICY_DISK_DELETING = "deleting" - SNAPSHOT_POLICY_DISK_DELETE_FAILED = "delete_failed" - INSTANCE_SNAPSHOT_READY = compute.INSTANCE_SNAPSHOT_READY INSTANCE_SNAPSHOT_UNKNOWN = "unknown" INSTANCE_SNAPSHOT_FAILED = "instance_snapshot_create_failed" @@ -54,10 +38,6 @@ const ( INSTANCE_SNAPSHOT_DELETE_FAILED = "instance_snapshot_delete_failed" INSTANCE_SNAPSHOT_RESET = "instance_snapshot_reset" - SNAPSHOT_POLICY_CACHE_STATUS_READY = "ready" - SNAPSHOT_POLICY_CACHE_STATUS_DELETING = "deleting" - SNAPSHOT_POLICY_CACHE_STATUS_DELETE_FAILED = "delete_failed" - SNAPSHOT_EXIST = "exist" SNAPSHOT_NOT_EXIST = "not_exist" ) diff --git a/pkg/apis/compute/snapshotpolicy.go b/pkg/apis/compute/snapshotpolicy.go index ba890c854bf..7ab3794504c 100644 --- a/pkg/apis/compute/snapshotpolicy.go +++ b/pkg/apis/compute/snapshotpolicy.go @@ -15,92 +15,77 @@ package compute import ( - "fmt" - "sort" + "reflect" "yunion.io/x/jsonutils" - "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/util/bitmap" +) + +const ( + SNAPSHOT_POLICY_APPLY = "applying" + SNAPSHOT_POLICY_APPLY_FAILED = "apply_failed" + SNAPSHOT_POLICY_CANCEL = "canceling" + SNAPSHOT_POLICY_CANCEL_FAILED = "cancel_failed" ) type SnapshotPolicyDetails struct { apis.VirtualResourceDetails + ManagedResourceInfo + CloudregionResourceInfo SSnapshotPolicy - RetentionDays int `json:"retention_days"` - RepeatWeekdaysDisplay []int `json:"repeat_weekdays_display"` - TimePointsDisplay []int `json:"time_points_display"` - IsActivated *bool `json:"is_activated,omitempty"` + RetentionDays int `json:"retention_days"` BindingDiskCount int `json:"binding_disk_count"` } -type SnapshotPolicyResourceInfo struct { - // 快照策略名称 - Snapshotpolicy string `json:"snapshotpolicy"` -} - type SSnapshotPolicyCreateInput struct { apis.VirtualResourceCreateInput + CloudproviderResourceInput + CloudregionResourceInput - RetentionDays int `json:"retention_days"` - RepeatWeekdays string `json:"repeat_weekdays"` - TimePoints string `json:"time_points"` -} + RetentionDays int `json:"retention_days"` -func (self SSnapshotPolicyCreateInput) GetRepeatWeekdays(limit int) (uint32, error) { - days, err := jsonutils.ParseString(self.RepeatWeekdays) - if err != nil { - return 0, errors.Wrapf(err, "ParseString %s", self.RepeatWeekdays) - } - repeatWeekdays := []int{} - days.Unmarshal(&repeatWeekdays) - err = daysCheck(repeatWeekdays, 1, 7) - if err != nil { - return 0, err - } - if len(repeatWeekdays) > limit { - return 0, httperrors.NewInputParameterError("repeat_weekdays only contains %d days at most", limit) - } - return bitmap.IntArray2Uint(repeatWeekdays), nil + RepeatWeekdays []int `json:"repeat_weekdays"` + + TimePoints []int `json:"time_points"` } -func (self SSnapshotPolicyCreateInput) GetTimePoints(limit int) (uint32, error) { - points, err := jsonutils.ParseString(self.TimePoints) - if err != nil { - return 0, errors.Wrapf(err, "ParseString %s", self.TimePoints) +func (self *SSnapshotPolicyCreateInput) Validate() error { + if len(self.RepeatWeekdays) == 0 { + return httperrors.NewMissingParameterError("repeat_weekdays") } - timepoints := []int{} - points.Unmarshal(&timepoints) - err = daysCheck(timepoints, 0, 23) - if err != nil { - return 0, err - } - if len(timepoints) > limit { - return 0, httperrors.NewInputParameterError("time_points only contains %d points at most", limit) - } - return bitmap.IntArray2Uint(timepoints), nil -} -func daysCheck(days []int, min, max int) error { - if len(days) == 0 { - return nil + repeatDays := []int{} + for _, day := range self.RepeatWeekdays { + if day < 1 || day > 7 { + return httperrors.NewOutOfRangeError("repeat_weekdays out of range 1-7") + } + if !utils.IsInArray(day, repeatDays) { + repeatDays = append(repeatDays, day) + } } - sort.Ints(days) + self.RepeatWeekdays = repeatDays - if days[0] < min || days[len(days)-1] > max { - return fmt.Errorf("Out of range") + if len(self.TimePoints) == 0 { + return httperrors.NewMissingParameterError("time_points") } - for i := 1; i < len(days); i++ { - if days[i] == days[i-1] { - return fmt.Errorf("Has repeat day %v", days) + points := []int{} + for _, point := range self.TimePoints { + if point < 0 || point > 23 { + return httperrors.NewOutOfRangeError("time_points out of range 0-23") + } + if !utils.IsInArray(point, points) { + points = append(points, point) } } + self.TimePoints = points return nil } @@ -109,46 +94,69 @@ type SSnapshotPolicyUpdateInput struct { RetentionDays *int - RepeatWeekdays *string `json:"repeat_weekdays"` - TimePoints *string `json:"time_points"` + RepeatWeekdays *[]int `json:"repeat_weekdays"` + TimePoints *[]int `json:"time_points"` } -func (self SSnapshotPolicyUpdateInput) GetRepeatWeekdays(limit int) (uint32, error) { +func (self *SSnapshotPolicyUpdateInput) Validate() error { if self.RepeatWeekdays != nil { - days, err := jsonutils.ParseString(*self.RepeatWeekdays) - if err != nil { - return 0, errors.Wrapf(err, "ParseString %s", *self.RepeatWeekdays) - } - repeatWeekdays := []int{} - days.Unmarshal(&repeatWeekdays) - err = daysCheck(repeatWeekdays, 1, 7) - if err != nil { - return 0, err + repeatDays := []int{} + for _, day := range *self.RepeatWeekdays { + if day < 1 || day > 7 { + return httperrors.NewOutOfRangeError("repeat_weekdays out of range 1-7") + } + if !utils.IsInArray(day, repeatDays) { + repeatDays = append(repeatDays, day) + } } - if len(repeatWeekdays) > limit { - return 0, httperrors.NewInputParameterError("repeat_weekdays only contains %d days at most", limit) - } - return bitmap.IntArray2Uint(repeatWeekdays), nil + self.RepeatWeekdays = &repeatDays } - return 0, nil -} -func (self SSnapshotPolicyUpdateInput) GetTimePoints(limit int) (uint32, error) { if self.TimePoints != nil { - points, err := jsonutils.ParseString(*self.TimePoints) - if err != nil { - return 0, errors.Wrapf(err, "ParseString %s", *self.TimePoints) + points := []int{} + for _, point := range *self.TimePoints { + if point < 0 || point > 23 { + return httperrors.NewOutOfRangeError("time_points out of range 0-23") + } + if !utils.IsInArray(point, points) { + points = append(points, point) + } } - timepoints := []int{} - points.Unmarshal(&timepoints) - err = daysCheck(timepoints, 0, 23) - if err != nil { - return 0, err - } - if len(timepoints) > limit { - return 0, httperrors.NewInputParameterError("time_points only contains %d points at most", limit) - } - return bitmap.IntArray2Uint(timepoints), nil + self.TimePoints = &points } - return 0, nil + return nil +} + +type SnapshotPolicyDisksInput struct { + Disks []string `json:"disk"` +} + +type RepeatWeekdays []int + +func (days RepeatWeekdays) String() string { + return jsonutils.Marshal(days).String() +} + +func (days RepeatWeekdays) IsZero() bool { + return len(days) == 0 +} + +type TimePoints []int + +func (points TimePoints) String() string { + return jsonutils.Marshal(points).String() +} + +func (points TimePoints) IsZero() bool { + return len(points) == 0 +} + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&RepeatWeekdays{}), func() gotypes.ISerializable { + return &RepeatWeekdays{} + }) + gotypes.RegisterSerializable(reflect.TypeOf(&TimePoints{}), func() gotypes.ISerializable { + return &TimePoints{} + }) + } diff --git a/pkg/apis/compute/snapshotpolicycache.go b/pkg/apis/compute/snapshotpolicycache.go deleted file mode 100644 index 03a2ba16762..00000000000 --- a/pkg/apis/compute/snapshotpolicycache.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import "yunion.io/x/onecloud/pkg/apis" - -type SnapshotPolicyCacheDetails struct { - apis.StatusStandaloneResourceDetails - ManagedResourceInfo - CloudregionResourceInfo - SnapshotPolicyResourceInfo - SSnapshotPolicyCache -} diff --git a/pkg/apis/compute/snapshotpolicydisk.go b/pkg/apis/compute/snapshotpolicydisk.go deleted file mode 100644 index e85517322e5..00000000000 --- a/pkg/apis/compute/snapshotpolicydisk.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import ( - "yunion.io/x/onecloud/pkg/apis" -) - -type SnapshotPolicyDiskDetails struct { - apis.VirtualJointResourceBaseDetails - - // 快照策略名称 - Snapshotpolicy string `json:"snapshotpolicy"` - Disk DiskDetails `json:"disk"` -} - -type SnapshotPolicyDiskListInput struct { - apis.VirtualJointResourceBaseListInput - SnapshotPolicyFilterListInput - DiskFilterListInput - - // 状态 - Status []string `json:"status"` -} diff --git a/pkg/apis/compute/zz_generated.model.go b/pkg/apis/compute/zz_generated.model.go index 5a7251c9037..85e4a4eda83 100644 --- a/pkg/apis/compute/zz_generated.model.go +++ b/pkg/apis/compute/zz_generated.model.go @@ -2663,8 +2663,7 @@ type SSnapshotPolicy struct { // 1~7, 1 is Monday RepeatWeekdays byte `json:"repeat_weekdays"` // 0~23 - TimePoints uint32 `json:"time_points"` - IsActivated *bool `json:"is_activated,omitempty"` + TimePoints uint32 `json:"time_points"` } // SSnapshotPolicyCache is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SSnapshotPolicyCache. diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index 50897da5af5..5afa01c6231 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -80,6 +80,9 @@ type SCapabilities struct { SecurityGroupBrands []string `json:",allowempty"` DisabledSecurityGroupBrands []string `json:",allowempty"` + SnapshotPolicyBrands []string `json:",allowempty"` + DisabledSnapshotPolicyBrands []string `json:",allowempty"` + ReadOnlyBrands []string `json:",allowempty"` ReadOnlyDisabledBrands []string `json:",allowempty"` ReadOnlyComputeEngineBrands []string `json:",allowempty"` @@ -119,6 +122,9 @@ type SCapabilities struct { ReadOnlySecurityGroupBrands []string `json:",allowempty"` ReadOnlyDisabledSecurityGroupBrands []string `json:",allowempty"` + ReadOnlySnapshotPolicyBrands []string `json:",allowempty"` + ReadOnlyDisabledSnapshotPolicyBrands []string `json:",allowempty"` + ResourceTypes []string `json:",allowempty"` StorageTypes []string `json:",allowempty"` // going to remove on 2.14 DataStorageTypes []string `json:",allowempty"` // going to remove on 2.14 @@ -374,6 +380,7 @@ func getBrands(region *SCloudregion, zone *SZone, domainId string, capa *SCapabi capa.Brands = append(capa.Brands, api.ONECLOUD_BRAND_ONECLOUD) capa.SecurityGroupBrands = append(capa.SecurityGroupBrands, api.ONECLOUD_BRAND_ONECLOUD) capa.ComputeEngineBrands = append(capa.ComputeEngineBrands, api.ONECLOUD_BRAND_ONECLOUD) + capa.SnapshotPolicyBrands = append(capa.SnapshotPolicyBrands, api.ONECLOUD_BRAND_ONECLOUD) } if count, _ := LoadbalancerClusterManager.Query().Limit(1).CountWithError(); count > 0 { @@ -463,6 +470,8 @@ func getBrands(region *SCloudregion, zone *SZone, domainId string, capa *SCapabi appendBrand(&capa.VpcPeerBrands, &capa.DisabledVpcPeerBrands, &capa.ReadOnlyVpcPeerBrands, &capa.ReadOnlyDisabledVpcPeerBrands, brand, capability, enabled, readOnly) case cloudprovider.CLOUD_CAPABILITY_SECURITY_GROUP: appendBrand(&capa.SecurityGroupBrands, &capa.DisabledSecurityGroupBrands, &capa.ReadOnlySecurityGroupBrands, &capa.ReadOnlyDisabledSecurityGroupBrands, brand, capability, enabled, readOnly) + case cloudprovider.CLOUD_CAPABILITY_SNAPSHOT_POLICY: + appendBrand(&capa.SnapshotPolicyBrands, &capa.DisabledSnapshotPolicyBrands, &capa.ReadOnlySnapshotPolicyBrands, &capa.ReadOnlyDisabledSnapshotPolicyBrands, brand, capability, enabled, readOnly) default: } } diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index 709a56130e2..9cf18acb63a 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -2018,7 +2018,7 @@ func syncRegionSnapshotPolicies( result := func() compare.SyncResult { defer syncResults.AddSqlCost(SnapshotPolicyManager)() - return SnapshotPolicyManager.SyncSnapshotPolicies(ctx, userCred, provider, localRegion, snapshotPolicies, provider.GetOwnerId(), syncRange.Xor) + return localRegion.SyncSnapshotPolicies(ctx, userCred, provider, snapshotPolicies, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(SnapshotPolicyManager, result) msg := result.Result() @@ -2172,10 +2172,6 @@ func syncPublicCloudProviderInfo( } if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { - if syncRange.IsNotSkipSyncResource(SnapshotPolicyManager) { - // sync snapshot policies before sync disks - syncRegionSnapshotPolicies(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) - } for j := 0; j < len(localZones); j += 1 { @@ -2194,6 +2190,11 @@ func syncPublicCloudProviderInfo( } } + if syncRange.IsNotSkipSyncResource(SnapshotPolicyManager) { + // sync snapshot policies after sync disks + syncRegionSnapshotPolicies(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) + } + // sync snapshots after sync disks if syncRange.IsNotSkipSyncResource(SnapshotManager) { syncRegionSnapshots(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 49ca4ea6c7b..5aa2032e051 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -31,7 +31,6 @@ import ( "yunion.io/x/pkg/util/billing" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/pinyinutils" - "yunion.io/x/pkg/util/rand" "yunion.io/x/pkg/util/rbacscope" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -1702,20 +1701,11 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke if err != nil { return errors.Wrapf(err, "db.UpdateWithLock") } - - // sync disk's snapshotpolicy - snapshotpolicies, err := extDisk.GetExtSnapshotPolicyIds() - if err != nil { - return errors.Wrapf(err, "Get snapshot policies of ICloudDisk %s.", extDisk.GetId()) - } storage, err := self.GetStorage() if err != nil { return errors.Wrapf(err, "GetStorage") } - err = SnapshotPolicyDiskManager.SyncByDisk(ctx, userCred, snapshotpolicies, syncOwnerId, self, storage) - if err != nil { - return err - } + db.OpsLog.LogSyncUpdate(self, diff, userCred) if len(diff) > 0 { @@ -1794,16 +1784,6 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl return nil, errors.Wrapf(err, "newFromCloudDisk") } - // create new joint model aboutsnapshotpolicy and disk - snapshotpolicies, err := extDisk.GetExtSnapshotPolicyIds() - if err != nil { - log.Warningln("GetExtSnapshotPolicyIds:", errors.Wrapf(err, "Get snapshot policies of ICloudDisk %s.", extDisk.GetId())) - } - err = SnapshotPolicyDiskManager.SyncAttachDiskExt(ctx, userCred, snapshotpolicies, syncOwnerId, &disk, storage) - if err != nil { - log.Warningln("SyncAttachDiskExt:", err) - } - syncVirtualResourceMetadata(ctx, userCred, &disk, extDisk, false) SyncCloudProject(ctx, userCred, &disk, syncOwnerId, extDisk, storage.ManagerId) @@ -2373,8 +2353,8 @@ func (manager *SDiskManager) FetchCustomizeColumns( Id string Name string Status string - TimePoints uint32 - RepeatWeekdays uint8 + TimePoints []int + RepeatWeekdays []int DiskId string }{} err = q.All(&policyInfo) @@ -2392,8 +2372,8 @@ func (manager *SDiskManager) FetchCustomizeColumns( policies[policy.DiskId] = append(policies[policy.DiskId], api.SimpleSnapshotPolicy{ Id: policy.Id, Name: policy.Name, - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysToIntArray(policy.RepeatWeekdays), - TimePoints: SnapshotPolicyManager.TimePointsToIntArray(policy.TimePoints), + RepeatWeekdays: policy.RepeatWeekdays, + TimePoints: policy.TimePoints, }) } @@ -2674,7 +2654,7 @@ func (manager *SDiskManager) CleanPendingDeleteDisks(ctx context.Context, userCr } } -func (manager *SDiskManager) getAutoSnapshotDisksId(isExternal bool) ([]SSnapshotPolicyDisk, error) { +func (manager *SDiskManager) GetNeedAutoSnapshotDisks() ([]SSnapshotPolicyDisk, error) { tz, _ := time.LoadLocation(options.Options.TimeZone) t := time.Now().In(tz) week := t.Weekday() @@ -2683,39 +2663,26 @@ func (manager *SDiskManager) getAutoSnapshotDisksId(isExternal bool) ([]SSnapsho } timePoint := t.Hour() - sps, err := SnapshotPolicyManager.GetSnapshotPoliciesAt(uint32(week), uint32(timePoint)) - if err != nil { - return nil, err - } - if len(sps) == 0 { - return nil, nil - } - - spds := make([]SSnapshotPolicyDisk, 0) - spdq := SnapshotPolicyDiskManager.Query() - spdq.NotEquals("status", api.SNAPSHOT_POLICY_DISK_INIT) - spdq.Filter(sqlchemy.In(spdq.Field("snapshotpolicy_id"), sps)) - - diskQ := DiskManager.Query().SubQuery() - spdq.Join(diskQ, sqlchemy.Equals(spdq.Field("disk_id"), diskQ.Field("id"))) - if !isExternal { - spdq.Filter(sqlchemy.IsNullOrEmpty(diskQ.Field("external_id"))) - } else { - spdq.Filter(sqlchemy.IsNotEmpty(diskQ.Field("external_id"))) - } - err = spdq.All(&spds) + policy := SnapshotPolicyManager.Query().Equals("cloudregion_id", api.DEFAULT_REGION_ID) + policy = policy.Filter(sqlchemy.Contains(policy.Field("repeat_weekdays"), fmt.Sprintf("%d", week))) + sq := policy.Filter( + sqlchemy.OR( + sqlchemy.Contains(policy.Field("time_points"), fmt.Sprintf(",%d,", timePoint)), + sqlchemy.Startswith(policy.Field("time_points"), fmt.Sprintf("[%d,", timePoint)), + sqlchemy.Endswith(policy.Field("time_points"), fmt.Sprintf(",%d]", timePoint)), + sqlchemy.Equals(policy.Field("time_points"), fmt.Sprintf("[%d]", timePoint)), + ), + ).SubQuery() + disks := DiskManager.Query().SubQuery() + q := SnapshotPolicyDiskManager.Query() + q = q.Join(sq, sqlchemy.Equals(q.Field("snapshotpolicy_id"), sq.Field("id"))) + q = q.Join(disks, sqlchemy.Equals(q.Field("disk_id"), disks.Field("id"))) + ret := []SSnapshotPolicyDisk{} + err := db.FetchModelObjects(SnapshotPolicyDiskManager, q, &ret) if err != nil { return nil, err } - return spds, nil -} - -func generateAutoSnapshotName() string { - name := "Auto-" + rand.String(8) - for SnapshotManager.Query().Equals("name", name).Count() > 0 { - name = "Auto-" + rand.String(8) - } - return name + return ret, nil } func (disk *SDisk) validateDiskAutoCreateSnapshot() error { @@ -2723,9 +2690,9 @@ func (disk *SDisk) validateDiskAutoCreateSnapshot() error { if len(guests) == 0 { return fmt.Errorf("Disks %s not attach guest, can't create snapshot", disk.GetName()) } - storage, _ := disk.GetStorage() - if storage == nil { - return fmt.Errorf("no valid storage") + storage, err := disk.GetStorage() + if err != nil { + return errors.Wrapf(err, "GetStorage") } if len(guests) == 1 && utils.IsInStringArray(storage.StorageType, api.FIEL_STORAGE) { if !utils.IsInStringArray(guests[0].Status, []string{api.VM_RUNNING, api.VM_READY}) { @@ -2736,86 +2703,124 @@ func (disk *SDisk) validateDiskAutoCreateSnapshot() error { } func (manager *SDiskManager) AutoDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - spds, err := manager.getAutoSnapshotDisksId(false) + disks, err := manager.GetNeedAutoSnapshotDisks() if err != nil { log.Errorf("Get auto snapshot disks id failed: %s", err) return } - if len(spds) == 0 { - log.Infof("CronJob AutoDiskSnapshot: No disk need create snapshot") - return - } + log.Debugf("auto snapshot %d disks", len(disks)) now := time.Now() - for i := 0; i < len(spds); i++ { - var ( - disk = manager.FetchDiskById(spds[i].DiskId) - snapshotPolicy, _ = SnapshotPolicyManager.FetchSnapshotPolicyById(spds[i].SnapshotpolicyId) - snapshotName = generateAutoSnapshotName() - autoSnapshotCount = options.Options.DefaultMaxSnapshotCount - options.Options.DefaultMaxManualSnapshotCount - err error - snapCount int - cleanOverdueSnapshots bool - ) - - if err = disk.validateDiskAutoCreateSnapshot(); err != nil { - goto onFail + for i := 0; i < len(disks); i++ { + disk, err := disks[i].GetDisk() + if err != nil { + log.Errorf("get disk error: %v", err) + continue } + autoSnapshotCount := options.Options.DefaultMaxSnapshotCount - options.Options.DefaultMaxManualSnapshotCount - if err = disk.CreateSnapshotAuto(ctx, userCred, snapshotName, snapshotPolicy); err != nil { - goto onFail - } + err = func() error { + policy, err := disks[i].GetSnapshotPolicy() + if err != nil { + return errors.Wrapf(err, "GetSnapshotPolicy") + } + + if len(disk.ExternalId) == 0 { + err = disk.validateDiskAutoCreateSnapshot() + if err != nil { + return errors.Wrapf(err, "validateDiskAutoCreateSnapshot") + } + } - snapCount, err = SnapshotManager.Query().Equals("fake_deleted", false). - Equals("disk_id", disk.Id).Equals("created_by", api.SNAPSHOT_AUTO). - CountWithError() + snapshot, err := disk.CreateSnapshotAuto(ctx, userCred, policy) + if err != nil { + return errors.Wrapf(err, "CreateSnapshotAuto") + } + + snapCount, err := SnapshotManager.Query().Equals("fake_deleted", false). + Equals("disk_id", disk.Id).Equals("created_by", api.SNAPSHOT_AUTO). + CountWithError() + if err != nil { + return errors.Wrap(err, "get snapshot count") + } + // if auto snapshot count gt max auto snapshot count, do clean overdued snapshots + cleanOverdueSnapshots := snapCount > autoSnapshotCount + if cleanOverdueSnapshots { + disk.CleanOverdueSnapshots(ctx, userCred, policy, now) + } + db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SNAPSHOT, snapshot.Name, userCred) + policy.ExecuteNotify(ctx, userCred, disk.GetName()) + return nil + }() if err != nil { - err = errors.Wrap(err, "get snapshot count") - goto onFail - } - // if auto snapshot count gt max auto snapshot count, do clean overdued snapshots - cleanOverdueSnapshots = snapCount > autoSnapshotCount - if cleanOverdueSnapshots { - disk.CleanOverdueSnapshots(ctx, userCred, snapshotPolicy, now) + log.Errorf("auto snapshot %s error: %v", disk.Name, err) + db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, err.Error(), userCred) + notifyclient.NotifySystemErrorWithCtx(ctx, disk.Id, disk.Name, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, errors.Wrapf(err, "Disk auto create snapshot").Error()) } - db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SNAPSHOT, "disk auto snapshot "+snapshotName, userCred) - snapshotPolicy.ExecuteNotify(ctx, userCred, disk.GetName()) - continue - onFail: - db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, err.Error(), userCred) - reason := fmt.Sprintf("Disk auto create snapshot failed: %s", err.Error()) - notifyclient.NotifySystemErrorWithCtx(ctx, disk.Id, disk.Name, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, reason) } + } func (self *SDisk) CreateSnapshotAuto( ctx context.Context, userCred mcclient.TokenCredential, - snapshotName string, snapshotPolicy *SSnapshotPolicy, -) error { - snap, err := SnapshotManager.CreateSnapshot(ctx, self.GetOwnerId(), api.SNAPSHOT_AUTO, - self.Id, "", "", snapshotName, snapshotPolicy.RetentionDays, false) + policy *SSnapshotPolicy, +) (*SSnapshot, error) { + storage, err := self.GetStorage() if err != nil { - return errors.Wrap(err, "disk create snapshot auto") + return nil, errors.Wrapf(err, "GetStorage") + } + snapshot := &SSnapshot{} + snapshot.SetModelManager(SnapshotManager, snapshot) + snapshot.ProjectId = self.ProjectId + snapshot.DomainId = self.DomainId + snapshot.DiskId = self.Id + if len(self.ExternalId) == 0 { + snapshot.StorageId = self.StorageId } - db.OpsLog.LogEvent(snap, db.ACT_CREATE, "disk create snapshot auto", userCred) - err = snap.StartSnapshotCreateTask(ctx, userCred, nil, "") + // inherit encrypt_key_id + snapshot.EncryptKeyId = self.EncryptKeyId + + driver, err := storage.GetRegionDriver() if err != nil { - return errors.Wrap(err, "disk auto snapshot start snapshot task") + return nil, errors.Wrapf(err, "GetRegionDriver") } - return nil + snapshot.OutOfChain = driver.SnapshotIsOutOfChain(self) + snapshot.Size = self.DiskSize + snapshot.DiskType = self.DiskType + snapshot.Location = "" + snapshot.CreatedBy = api.SNAPSHOT_AUTO + snapshot.ManagerId = storage.ManagerId + if cloudregion, _ := storage.GetRegion(); cloudregion != nil { + snapshot.CloudregionId = cloudregion.GetId() + } + snapshot.Name = fmt.Sprintf("%s-auto-snapshot-%d", self.Name, time.Now().Unix()) + snapshot.Status = api.SNAPSHOT_CREATING + if policy.RetentionDays > 0 { + snapshot.ExpiredAt = time.Now().AddDate(0, 0, policy.RetentionDays) + } + snapshot.IsSystem = self.IsSystem + err = SnapshotManager.TableSpec().Insert(ctx, snapshot) + if err != nil { + return nil, errors.Wrapf(err, "Insert") + } + + db.OpsLog.LogEvent(snapshot, db.ACT_CREATE, "disk create snapshot auto", userCred) + err = snapshot.StartSnapshotCreateTask(ctx, userCred, nil, "") + if err != nil { + return nil, errors.Wrap(err, "disk auto snapshot start snapshot task") + } + return snapshot, nil } func (self *SDisk) CleanOverdueSnapshots(ctx context.Context, userCred mcclient.TokenCredential, sp *SSnapshotPolicy, now time.Time) error { kwargs := jsonutils.NewDict() - kwargs.Set("snapshotpolicy_id", jsonutils.NewString(sp.Id)) + kwargs.Set("retention_day", jsonutils.NewInt(int64(sp.RetentionDays))) kwargs.Set("start_time", jsonutils.NewTimeString(now)) - if task, err := taskman.TaskManager.NewTask(ctx, "DiskCleanOverduedSnapshots", self, userCred, kwargs, "", "", nil); err != nil { - log.Errorln(err) - return err - } else { - task.ScheduleRun(nil) + task, err := taskman.TaskManager.NewTask(ctx, "DiskCleanOverduedSnapshots", self, userCred, kwargs, "", "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") } - return nil + return task.ScheduleRun(nil) } func (self *SDisk) StartCreateBackupTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { @@ -3003,85 +3008,48 @@ func (disk *SDisk) GetUsages() []db.IUsage { } } -func (disk *SDisk) PerformBindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - - spIden, err := data.GetString("snapshotpolicy") - if err != nil { - return nil, httperrors.NewMissingParameterError("miss snapshotpolicy") - } - // check snapshotpolicy - imodel, err := db.FetchByIdOrName(SnapshotPolicyManager, userCred, spIden) - if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewInputParameterError("no such snapshotpolicy %s", spIden) - } +func (disk *SDisk) PerformBindSnapshotpolicy( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.DiskSnapshotpolicyInput, +) (jsonutils.JSONObject, error) { + spObj, err := validators.ValidateModel(userCred, SnapshotPolicyManager, &input.SnapshotpolicyId) if err != nil { - return nil, errors.Wrap(err, "db.FetchByIdOrName") + return nil, err } - snapshotpolicy := imodel.(*SSnapshotPolicy) - - // try to bind - spd, err := SnapshotPolicyDiskManager.newSnapshotpolicyDisk(ctx, userCred, snapshotpolicy, disk) - - if errors.Cause(err) == ErrExistSD { - if spd.Status != api.SNAPSHOT_POLICY_DISK_INIT { - return nil, nil + sp := spObj.(*SSnapshotPolicy) + if len(sp.ManagerId) > 0 { + storage, err := disk.GetStorage() + if err != nil { + return nil, errors.Wrapf(err, "GetStorage") + } + if storage.ManagerId != sp.ManagerId { + return nil, httperrors.NewConflictError("The snapshot policy %s and disk account are different", sp.Name) + } + zone, err := storage.GetZone() + if err != nil { + return nil, errors.Wrapf(err, "GetZone") + } + if sp.CloudregionId != zone.CloudregionId { + return nil, httperrors.NewConflictError("The snapshot policy %s and the disk are in different region", sp.Name) } - } else if err != nil { - return nil, errors.Wrap(err, "SnapshotPolicyDiskManager.newSnapshotpolicyDisk") - } - - // start up SnapshotPolicyApplyTask - taskData := jsonutils.NewDict() - taskData.Add(jsonutils.Marshal(spd), "snapshotPolicyDisk") - taskData.Add(jsonutils.Marshal(snapshotpolicy), "snapshotPolicy") - if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyApplyTask", disk, userCred, nil, "", "", - nil); err != nil { - return nil, errors.Wrap(err, "fail to start up SnapshotPolicyApplyTask") - } else { - task.ScheduleRun(taskData) } - return nil, nil + return nil, sp.StartBindDisksTask(ctx, userCred, []string{disk.Id}) } -func (disk *SDisk) PerformUnbindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - - spIden, err := data.GetString("snapshotpolicy") - if err != nil { - return nil, httperrors.NewMissingParameterError("miss snapshotpolicy") - } - // check snapshotpolicy - imodel, err := db.FetchByIdOrName(SnapshotPolicyManager, userCred, spIden) - if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewInputParameterError("no such snapshotpolicy %s", spIden) - } - if err != nil { - return nil, errors.Wrap(err, "db.FetchByIdOrName") - } - snapshotpolicy := imodel.(*SSnapshotPolicy) - - spd, err := SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(snapshotpolicy.GetId(), disk.GetId()) +func (disk *SDisk) PerformUnbindSnapshotpolicy( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.DiskSnapshotpolicyInput, +) (jsonutils.JSONObject, error) { + spObj, err := validators.ValidateModel(userCred, SnapshotPolicyManager, &input.SnapshotpolicyId) if err != nil { - return nil, errors.Wrap(err, "SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk") - } - if spd == nil { - // has been detach - return nil, nil - } - - // start up SnapshotPolicyCancelTask - taskdata := jsonutils.NewDict() - taskdata.Add(jsonutils.NewString(snapshotpolicy.Id), "snapshot_policy_id") - taskdata.Add(jsonutils.Marshal(spd), "snapshotPolicyDisk") - if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCancelTask", disk, userCred, nil, "", "", - nil); err != nil { - return nil, errors.Wrap(err, "fail to start up SnapshotPolicyCancelTask") - } else { - spd.SetStatus(userCred, api.SNAPSHOT_POLICY_DISK_DELETING, "") - task.ScheduleRun(taskdata) + return nil, err } - return nil, nil + sp := spObj.(*SSnapshotPolicy) + return nil, sp.StartUnbindDisksTask(ctx, userCred, []string{disk.Id}) } func (manager *SDiskManager) ListItemExportKeys(ctx context.Context, diff --git a/pkg/compute/models/purge.go b/pkg/compute/models/purge.go index 569a054cdca..bc04910f51b 100644 --- a/pkg/compute/models/purge.go +++ b/pkg/compute/models/purge.go @@ -278,7 +278,7 @@ func (self *SCloudregion) purgeResources(ctx context.Context, managerId string) nicips := NetworkinterfacenetworkManager.Query("row_id").In("networkinterface_id", nics.SubQuery()) secgroups := SecurityGroupManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) rules := SecurityGroupRuleManager.Query("id").In("secgroup_id", secgroups.SubQuery()) - policycaches := SnapshotPolicyCacheManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) + policies := SnapshotPolicyManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) snapshots := SnapshotManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) tables := TablestoreManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) wafs := WafInstanceManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) @@ -295,7 +295,7 @@ func (self *SCloudregion) purgeResources(ctx context.Context, managerId string) {manager: WafInstanceManager, key: "id", q: wafs}, {manager: TablestoreManager, key: "id", q: tables}, {manager: SnapshotManager, key: "id", q: snapshots}, - {manager: SnapshotPolicyCacheManager, key: "id", q: policycaches}, + {manager: SnapshotPolicyManager, key: "id", q: policies}, {manager: SecurityGroupRuleManager, key: "id", q: rules}, {manager: SecurityGroupManager, key: "id", q: secgroups}, {manager: NetworkinterfacenetworkManager, key: "row_id", q: nicips}, diff --git a/pkg/compute/models/regiondrivers.go b/pkg/compute/models/regiondrivers.go index 88e58c6fd75..55972f75c7e 100644 --- a/pkg/compute/models/regiondrivers.go +++ b/pkg/compute/models/regiondrivers.go @@ -82,13 +82,11 @@ type IRegionDriver interface { type ISnapshotDriver interface { // Region Driver Snapshot Policy Apis - RequestUpdateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, sp *SSnapshotPolicy, input cloudprovider.SnapshotPolicyInput, task taskman.ITask) error - RequestApplySnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *SDisk, sp *SSnapshotPolicy, data jsonutils.JSONObject) error - RequestCancelSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *SDisk, sp *SSnapshotPolicy, data jsonutils.JSONObject) error - RequestPreSnapshotPolicyApply(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *SDisk, sp *SSnapshotPolicy, data jsonutils.JSONObject) error - - // Region Driver Snapshot Policy joint Disk Apis - ValidateCreateSnapshopolicyDiskData(ctx context.Context, userCred mcclient.TokenCredential, disk *SDisk, snapshotPolicy *SSnapshotPolicy) error + ValidateCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, input *api.SSnapshotPolicyCreateInput) (*api.SSnapshotPolicyCreateInput, error) + RequestCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, sp *SSnapshotPolicy, task taskman.ITask) error + RequestSnapshotPolicyBindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *SSnapshotPolicy, diskIds []string, task taskman.ITask) error + RequestSnapshotPolicyUnbindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *SSnapshotPolicy, diskIds []string, task taskman.ITask) error + RequestDeleteSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, sp *SSnapshotPolicy, task taskman.ITask) error // Region Driver Snapshot Apis ValidateSnapshotDelete(ctx context.Context, snapshot *SSnapshot) error diff --git a/pkg/compute/models/snapshotpolicy.go b/pkg/compute/models/snapshotpolicy.go index d321b8ec080..28dc338d7f9 100644 --- a/pkg/compute/models/snapshotpolicy.go +++ b/pkg/compute/models/snapshotpolicy.go @@ -17,45 +17,48 @@ package models import ( "context" "fmt" - "sort" - "time" + "strings" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/bitmap" - "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) type SSnapshotPolicyManager struct { db.SVirtualResourceBaseManager + db.SExternalizedResourceBaseManager + SManagedResourceBaseManager + SCloudregionResourceBaseManager } type SSnapshotPolicy struct { db.SVirtualResourceBase + db.SExternalizedResourceBase + SManagedResourceBase + + SCloudregionResourceBase `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"domain_required" default:"default"` RetentionDays int `nullable:"false" list:"user" get:"user" create:"required"` // 1~7, 1 is Monday - RepeatWeekdays uint8 `charset:"utf8" create:"required" list:"user" get:"user"` + RepeatWeekdays api.RepeatWeekdays `charset:"utf8" create:"required" list:"user" get:"user"` // 0~23 - TimePoints uint32 `charset:"utf8" create:"required" list:"user" get:"user"` - IsActivated tristate.TriState `list:"user" get:"user" create:"optional" default:"true"` + TimePoints api.TimePoints `charset:"utf8" create:"required" list:"user" get:"user"` } var SnapshotPolicyManager *SSnapshotPolicyManager @@ -64,7 +67,7 @@ func init() { SnapshotPolicyManager = &SSnapshotPolicyManager{ SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( SSnapshotPolicy{}, - "snapshotpolicies_tbl", + "snapshot_policies_tbl", "snapshotpolicy", "snapshotpolicies", ), @@ -72,96 +75,6 @@ func init() { SnapshotPolicyManager.SetVirtualObject(SnapshotPolicyManager) } -func (manager *SSnapshotPolicyManager) ValidateListConditions(ctx context.Context, userCred mcclient.TokenCredential, - query *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - - input := &api.SSnapshotPolicyCreateInput{} - err := query.Unmarshal(input) - if err != nil { - return nil, httperrors.NewInputParameterError("Unmarshal input failed %s", err) - } - if query.Contains("repeat_weekdays") { - repeat, err := input.GetRepeatWeekdays(options.Options.RepeatWeekdaysLimit) - if err != nil { - return nil, err - } - query.Set("repeat_weekdays", jsonutils.NewInt(int64(repeat))) - } - if query.Contains("time_points") { - timepoints, err := input.GetTimePoints(options.Options.TimePointsLimit) - if err != nil { - return nil, err - } - query.Set("time_points", jsonutils.NewInt(int64(timepoints))) - } - return query, nil -} - -func (manager *SSnapshotPolicyManager) CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*db.CustomizeListFilters, error) { - filters := db.NewCustomizeListFilters() - filters.Append(func(item jsonutils.JSONObject) (bool, error) { - itemDict, ok := item.(*jsonutils.JSONDict) - if !ok { - return false, nil - } - if days, err := itemDict.Int("repeat_weekdays"); err == nil { - newDays := manager.RepeatWeekdaysToIntArray(uint8(days)) - itemDict.Set("repeat_weekdays", jsonutils.Marshal(newDays)) - } - if tpoints, err := itemDict.Int("time_points"); err == nil { - newPoints := manager.TimePointsToIntArray(uint32(tpoints)) - itemDict.Set("time_points", jsonutils.Marshal(newPoints)) - } - return true, nil - }) - return filters, nil -} - -// ==================================================== fetch ========================================================== -func (manager *SSnapshotPolicyManager) GetSnapshotPoliciesAt(week, timePoint uint32) ([]string, error) { - - q := manager.Query("id") - q = q.Filter(sqlchemy.Equals(sqlchemy.AND_Val("", q.Field("repeat_weekdays"), 1< 0 { - ret := make([]string, len(sps)) - for i := 0; i < len(sps); i++ { - ret[i] = sps[i].Id - } - return ret, nil - } - return nil, nil -} - -func (manager *SSnapshotPolicyManager) FetchSnapshotPolicyById(spId string) (*SSnapshotPolicy, error) { - sp, err := manager.FetchById(spId) - if err != nil { - return nil, err - } - return sp.(*SSnapshotPolicy), nil -} - -func (manager *SSnapshotPolicyManager) FetchAllByIds(spIds []string) ([]SSnapshotPolicy, error) { - if spIds == nil || len(spIds) == 0 { - return []SSnapshotPolicy{}, nil - } - q := manager.Query().In("id", spIds) - sps := make([]SSnapshotPolicy, 0, 1) - err := db.FetchModelObjects(manager, q, &sps) - if err != nil { - return nil, err - } - return sps, nil -} - func (manager *SSnapshotPolicyManager) ValidateCreateData( ctx context.Context, userCred mcclient.TokenCredential, @@ -179,45 +92,39 @@ func (manager *SSnapshotPolicyManager) ValidateCreateData( return nil, err } - if len(input.RepeatWeekdays) == 0 { - return nil, httperrors.NewMissingParameterError("repeat_weekdays") - } + input.Status = apis.STATUS_CREATING - repeatDays, err := input.GetRepeatWeekdays(options.Options.RepeatWeekdaysLimit) + if len(input.CloudregionId) == 0 { + input.CloudregionId = api.DEFAULT_REGION_ID + } + regionObj, err := validators.ValidateModel(userCred, CloudregionManager, &input.CloudregionId) if err != nil { - return nil, httperrors.NewInputParameterError("%v", err) + return nil, err } + region := regionObj.(*SCloudregion) - input.RepeatWeekdays = fmt.Sprintf("%d", repeatDays) - - if len(input.TimePoints) == 0 { - return nil, httperrors.NewMissingParameterError("time_points") + input, err = region.GetDriver().ValidateCreateSnapshotPolicy(ctx, userCred, region, input) + if err != nil { + return nil, err } - timepoints, err := input.GetTimePoints(options.Options.TimePointsLimit) + err = input.Validate() if err != nil { - return nil, httperrors.NewInputParameterError("%v", err) + return nil, err } - - input.TimePoints = fmt.Sprintf("%d", timepoints) return input, nil } -func (manager *SSnapshotPolicyManager) OnCreateComplete(ctx context.Context, items []db.IModel, - userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data []jsonutils.JSONObject) { - for i := range items { - sp := items[i].(*SSnapshotPolicy) - db.Update(sp, func() error { - sp.Status = api.SNAPSHOT_POLICY_READY - return nil - }) - db.OpsLog.LogEvent(sp, db.ACT_DELETE, sp.GetShortDesc(ctx), userCred) - logclient.AddActionLogWithContext(ctx, sp, logclient.ACT_DELOCATE, nil, userCred, true) - notifyclient.EventNotify(ctx, userCred, notifyclient.SEventNotifyParam{ - Obj: sp, - Action: notifyclient.ActionDelete, - }) +func (sp *SSnapshotPolicy) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + sp.StartCreateTask(ctx, userCred) +} + +func (sp *SSnapshotPolicy) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential) error { + task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCreateTask", sp, userCred, nil, "", "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") } + return task.ScheduleRun(nil) } func (self *SSnapshotPolicy) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.SSnapshotPolicyUpdateInput) (*api.SSnapshotPolicyUpdateInput, error) { @@ -228,85 +135,32 @@ func (self *SSnapshotPolicy) ValidateUpdateData(ctx context.Context, userCred mc } if input.RetentionDays != nil { - if *input.RetentionDays < -1 || *input.RetentionDays > 65535 { - return nil, httperrors.NewInputParameterError("Retention days must in 1~65535 or -1") + if *input.RetentionDays < -1 || *input.RetentionDays == 0 || *input.RetentionDays > options.Options.RetentionDaysLimit { + return nil, httperrors.NewInputParameterError("Retention days must in 1~%d or -1", options.Options.RetentionDaysLimit) } } - if input.RepeatWeekdays != nil { - weekDays, err := input.GetRepeatWeekdays(options.Options.RepeatWeekdaysLimit) - if err != nil { - return nil, err - } - day := fmt.Sprintf("%d", weekDays) - input.RepeatWeekdays = &day - } - - if input.TimePoints != nil { - timepoint, err := input.GetTimePoints(options.Options.TimePointsLimit) - if err != nil { - return nil, err - } - points := fmt.Sprintf("%d", timepoint) - input.TimePoints = &points + err = input.Validate() + if err != nil { + return nil, err } return input, nil } -func (self *SSnapshotPolicy) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { - self.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data) - - input := &api.SSnapshotPolicyUpdateInput{} - data.Unmarshal(input) - - self.StartSnapshotPolicyUpdateTask(ctx, userCred, input) +func (sp *SSnapshotPolicy) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + return sp.StartDeleteTask(ctx, userCred) } -func (sp *SSnapshotPolicy) StartSnapshotPolicyUpdateTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.SSnapshotPolicyUpdateInput) error { - params := jsonutils.NewDict() - params.Add(jsonutils.Marshal(input), "input") - sp.SetStatus(userCred, api.SNAPSHOT_POLICY_UPDATING, "") - task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyUpdateTask", sp, userCred, params, "", "", nil) +func (sp *SSnapshotPolicy) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential) error { + sp.SetStatus(userCred, apis.STATUS_DELETING, "") + task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyDeleteTask", sp, userCred, nil, "", "", nil) if err != nil { return errors.Wrapf(err, "NewTask") } return task.ScheduleRun(nil) } -func (sp *SSnapshotPolicy) DetachAfterDelete(ctx context.Context, userCred mcclient.TokenCredential) error { - err := SnapshotPolicyDiskManager.SyncDetachBySnapshotpolicy(ctx, userCred, nil, sp) - if err != nil { - return errors.Wrap(err, "detach after delete failed") - } - return nil -} - -func (sp *SSnapshotPolicy) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - // check if sp bind to some disks - count, err := SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id) - if err != nil { - return errors.Wrap(err, "unable to FetchDiskCountBySPID") - } - if count != 0 { - return httperrors.NewBadRequestError("Couldn't delete snapshot policy binding to disks") - } - sp.SetStatus(userCred, api.SNAPSHOT_POLICY_DELETING, "") - return sp.StartSnapshotPolicyDeleteTask(ctx, userCred, jsonutils.NewDict(), "") -} - -func (sp *SSnapshotPolicy) StartSnapshotPolicyDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, - params *jsonutils.JSONDict, parentTaskId string) error { - - task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyDeleteTask", sp, userCred, params, - parentTaskId, "", nil) - if err != nil { - return err - } - task.ScheduleRun(nil) - return nil -} - func (manager *SSnapshotPolicyManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, @@ -318,12 +172,37 @@ func (manager *SSnapshotPolicyManager) FetchCustomizeColumns( rows := make([]api.SnapshotPolicyDetails, len(objs)) virtRows := manager.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + regionRows := manager.SCloudregionResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + policyIds := make([]string, len(objs)) for i := range rows { rows[i] = api.SnapshotPolicyDetails{ - VirtualResourceDetails: virtRows[i], + VirtualResourceDetails: virtRows[i], + ManagedResourceInfo: manRows[i], + CloudregionResourceInfo: regionRows[i], } - rows[i] = objs[i].(*SSnapshotPolicy).getMoreDetails(rows[i]) + policy := objs[i].(*SSnapshotPolicy) + policyIds[i] = policy.Id + } + + q := SnapshotPolicyDiskManager.Query().In("snapshotpolicy_id", policyIds) + pds := []SSnapshotPolicyDisk{} + err := q.All(&pds) + if err != nil { + return rows + } + pdMap := map[string][]SSnapshotPolicyDisk{} + for _, pd := range pds { + _, ok := pdMap[pd.SnapshotpolicyId] + if !ok { + pdMap[pd.SnapshotpolicyId] = []SSnapshotPolicyDisk{} + } + pdMap[pd.SnapshotpolicyId] = append(pdMap[pd.SnapshotpolicyId], pd) + } + for i := range rows { + res, _ := pdMap[policyIds[i]] + rows[i].BindingDiskCount = len(res) } return rows @@ -339,328 +218,172 @@ func (sp *SSnapshotPolicy) ExecuteNotify(ctx context.Context, userCred mcclient. }) } -func (sp *SSnapshotPolicy) getMoreDetails(out api.SnapshotPolicyDetails) api.SnapshotPolicyDetails { - out.RepeatWeekdaysDisplay = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays) - out.TimePointsDisplay = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints) - out.BindingDiskCount, _ = SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id) - return out -} - -func (sp *SSnapshotPolicy) GetCloudproviderId() string { - return "" +func (self *SCloudregion) GetSnapshotPolicies(managerId string) ([]SSnapshotPolicy, error) { + q := SnapshotPolicyManager.Query().Equals("cloudregion_id", self.Id) + if len(managerId) > 0 { + q = q.Equals("manager_id", managerId) + } + ret := []SSnapshotPolicy{} + err := db.FetchModelObjects(SnapshotPolicyManager, q, &ret) + if err != nil { + return nil, err + } + return ret, nil } -func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies( +func (region *SCloudregion) SyncSnapshotPolicies( ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, - region *SCloudregion, - cloudSPs []cloudprovider.ICloudSnapshotPolicy, + policies []cloudprovider.ICloudSnapshotPolicy, syncOwnerId mcclient.IIdentityProvider, xor bool, ) compare.SyncResult { + lockman.LockRawObject(ctx, SnapshotPolicyManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, SnapshotPolicyManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + result := compare.SyncResult{} - lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) - syncResult := compare.SyncResult{} - - // Fetch allsnapshotpolicy caches - spCaches, err := SnapshotPolicyCacheManager.FetchAllByRegionProvider(region.GetId(), provider.GetId()) + dbPolicies, err := region.GetSnapshotPolicies(provider.Id) if err != nil { - syncResult.Error(err) - return syncResult - } - spIdSet, spIds := make(map[string]struct{}), make([]string, 0, 2) - for _, spCache := range spCaches { - if _, ok := spIdSet[spCache.SnapshotpolicyId]; !ok { - spIds = append(spIds, spCache.SnapshotpolicyId) - spIdSet[spCache.SnapshotpolicyId] = struct{}{} - } + result.Error(err) + return result } - // Fetch allsnapshotpolicy of caches above - snapshotPolicies, err := manager.FetchAllByIds(spIds) - if err != nil { - syncResult.Error(err) - return syncResult - } - - // structure two sets (externalID, snapshotpolicyCache), (snapshotPolicyID, snapshotPolicy) - spSet, spCacheSet := make(map[string]*SSnapshotPolicy), make(map[string]*SSnapshotPolicyCache) - for i := range snapshotPolicies { - spSet[snapshotPolicies[i].GetId()] = &snapshotPolicies[i] - } - for i := range spCaches { - externalId := spCaches[i].ExternalId - if len(externalId) != 0 { - spCacheSet[spCaches[i].ExternalId] = &spCaches[i] - } - } - - // start sync - // add forsnapshotpolicy and cache - // delete for snapshotpolicy cache added := make([]cloudprovider.ICloudSnapshotPolicy, 0, 1) commonext := make([]cloudprovider.ICloudSnapshotPolicy, 0, 1) - commondb := make([]*SSnapshotPolicyCache, 0, 1) - removed := make([]*SSnapshotPolicyCache, 0, 1) - for _, cloudSP := range cloudSPs { - spCache, ok := spCacheSet[cloudSP.GetGlobalId()] - if !ok { - added = append(added, cloudSP) - continue - } - snapshotPolicy := spSet[spCache.SnapshotpolicyId] - if !snapshotPolicy.Equals(cloudSP) { - removed = append(removed, spCache) - added = append(added, cloudSP) - } else { - commondb = append(commondb, spCache) - commonext = append(commonext, cloudSP) - } - delete(spCacheSet, cloudSP.GetGlobalId()) - } + commondb := make([]SSnapshotPolicy, 0, 1) + removed := make([]SSnapshotPolicy, 0, 1) - for _, v := range spCacheSet { - removed = append(removed, v) + err = compare.CompareSets(dbPolicies, policies, &removed, &commondb, &commonext, &added) + if err != nil { + result.Error(err) + return result } - for i := range removed { - // changesnapshotpolicy cache - err := removed[i].RealDetele(ctx, userCred) + for i := 0; i < len(removed); i += 1 { + err = removed[i].RealDelete(ctx, userCred) if err != nil { - syncResult.DeleteError(err) + result.DeleteError(err) + continue } + result.Delete() } - syncResult = manager.allNewFromCloudSnapshotPolicy(ctx, userCred, added, region, syncOwnerId, provider, syncResult) - - if !xor { - for i := range commondb { - _, err = db.Update(commondb[i], func() error { - commondb[i].Status = api.SNAPSHOT_POLICY_CACHE_STATUS_READY - if len(commonext[i].GetName()) == 0 { - commondb[i].Name = commonext[i].GetId() - } else { - commondb[i].Name = commonext[i].GetName() - } - return nil - }) + for i := 0; i < len(commondb); i += 1 { + if !xor { + err = commondb[i].SyncWithCloudPolicy(ctx, userCred, provider, commonext[i]) if err != nil { - syncResult.UpdateError(err) + result.UpdateError(err) continue } - syncResult.Update() } + result.Update() } - return syncResult -} - -func (manager *SSnapshotPolicyManager) allNewFromCloudSnapshotPolicy( - ctx context.Context, userCred mcclient.TokenCredential, added []cloudprovider.ICloudSnapshotPolicy, - region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, - syncResult compare.SyncResult) compare.SyncResult { - var snapshotpolicyCluster map[uint64][]*SSnapshotPolicy - - if len(added) > 5 { - // the number of added is large - // fetch all snapshotpolicy - q := SnapshotPolicyManager.Query() - allSnapshotPolicies := make([]SSnapshotPolicy, 0, 10) - err := q.All(&allSnapshotPolicies) + for i := 0; i < len(added); i += 1 { + _, err := region.newFromCloudPolicy(ctx, userCred, provider, added[i]) if err != nil { - syncResult.Error(err) - return syncResult - } - // cluster snapshotpolicy - snapshotpolicyCluster := make(map[uint64][]*SSnapshotPolicy) - for i := range allSnapshotPolicies { - key := allSnapshotPolicies[i].Key() - list, ok := snapshotpolicyCluster[key] - if !ok { - list = make([]*SSnapshotPolicy, 0, 1) - } - list = append(list, &allSnapshotPolicies[i]) - // sliceHeader change - snapshotpolicyCluster[key] = list + result.AddError(err) + continue } + result.Add() } - for i := range added { - local, err := manager.newFromCloudSnapshotPolicy(ctx, userCred, snapshotpolicyCluster, added[i], region, - syncOwnerId, provider) - if err != nil { - syncResult.AddError(err) - } else { - syncVirtualResourceMetadata(ctx, userCred, local, added[i], false) - syncResult.Add() - } - } - return syncResult + return result } -func (manager *SSnapshotPolicyManager) newFromCloudSnapshotPolicy( - ctx context.Context, userCred mcclient.TokenCredential, snapshotpolicyCluster map[uint64][]*SSnapshotPolicy, - ext cloudprovider.ICloudSnapshotPolicy, region *SCloudregion, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, -) (*SSnapshotPolicy, error) { - snapshotPolicyTmp := SSnapshotPolicy{} - snapshotPolicyTmp.RetentionDays = ext.GetRetentionDays() - arw, err := ext.GetRepeatWeekdays() - if err != nil { - return nil, err - } - snapshotPolicyTmp.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysParseIntArray(arw) - atp, err := ext.GetTimePoints() - if err != nil { - return nil, err - } - snapshotPolicyTmp.TimePoints = SnapshotPolicyManager.TimePointsParseIntArray(atp) - snapshotPolicyTmp.IsActivated = tristate.NewFromBool(ext.IsActivated()) - - extProjectId := SnapshotPolicyManager.FetchProjectId(ctx, userCred, syncOwnerId, ext, provider.GetId()) - - var snapshotPolicy *SSnapshotPolicy - - // find suitable snapshotpolicy - if snapshotpolicyCluster == nil { - q := manager.Query().Equals("repeat_weekdays", snapshotPolicyTmp.RepeatWeekdays).Equals("time_points", - snapshotPolicyTmp.TimePoints).Equals("retention_days", snapshotPolicyTmp.RetentionDays).Equals( - "is_activated", snapshotPolicyTmp.IsActivated.Bool()).Equals("tenant_id", extProjectId) - count, err := q.CountWithError() - if err != nil { - return nil, err - } - if count > 0 { - snapshotPolicy = &SSnapshotPolicy{} - err = q.First(snapshotPolicy) - if err != nil { - return nil, err - } - snapshotPolicy.SetModelManager(manager, snapshotPolicy) - } - } else { - extkey := snapshotPolicyTmp.Key() - if list, ok := snapshotpolicyCluster[extkey]; ok { - // find first snapshotpolicy enough to rebase - for _, sp := range list { - if sp.ProjectId == extProjectId { - snapshotPolicy = sp - break - } +func (self *SSnapshotPolicy) SyncWithCloudPolicy( + ctx context.Context, userCred mcclient.TokenCredential, + provider *SCloudprovider, + ext cloudprovider.ICloudSnapshotPolicy, +) error { + _, err := db.Update(self, func() error { + if options.Options.EnableSyncName { + newName, _ := db.GenerateAlterName(self, ext.GetName()) + if len(newName) > 0 { + self.Name = newName } } - } - - // no such suitable snapshotpolicy - if snapshotPolicy == nil { - snapshotPolicyTmp.SetModelManager(manager, &snapshotPolicyTmp) - snapshotPolicyTmp.Status = ext.GetStatus() - - var err = func() error { - lockman.LockRawObject(ctx, manager.Keyword(), "name") - defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name") - newName, err := db.GenerateName(ctx, manager, syncOwnerId, ext.GetName()) - if err != nil { - return err - } - snapshotPolicyTmp.Name = newName - - return manager.TableSpec().Insert(ctx, &snapshotPolicyTmp) - }() + self.RetentionDays = ext.GetRetentionDays() + var err error + self.RepeatWeekdays, err = ext.GetRepeatWeekdays() if err != nil { - return nil, errors.Wrapf(err, "Insert") + return errors.Wrapf(err, "GetRepeatWeekdays") } - // sync project - SyncCloudProject(ctx, userCred, &snapshotPolicyTmp, syncOwnerId, ext, provider.GetId()) - // update snapshotpolicyCluster - if snapshotpolicyCluster != nil { - key := snapshotPolicyTmp.Key() - list, ok := snapshotpolicyCluster[key] - if !ok { - list = make([]*SSnapshotPolicy, 0) - } - list = append(list, &snapshotPolicyTmp) - snapshotpolicyCluster[key] = list + self.TimePoints, err = ext.GetTimePoints() + if err != nil { + return errors.Wrapf(err, "GetTimePoints") } - snapshotPolicy = &snapshotPolicyTmp - } - - // add cache - _, err = SnapshotPolicyCacheManager.NewCacheWithExternalId(ctx, userCred, snapshotPolicy.GetId(), - ext.GetGlobalId(), region.GetId(), provider.GetId(), ext.GetName()) + self.Status = ext.GetStatus() + return nil + }) if err != nil { - //snapshotpolicy has been exist so that created is successful although cache created is fail. - // disk will be sync aftersnapshotpolicy sync, cache must be right so that this sync is fail - log.Errorf("snapshotpolicy %s created successfully but corresponding cache created fail", snapshotPolicy.GetId()) - return nil, errors.Wrapf(err, "snapshotpolicy %s created successfully but corresponding cache created fail", - snapshotPolicy.GetId()) + return errors.Wrapf(err, "Update") } - db.OpsLog.LogEvent(snapshotPolicy, db.ACT_CREATE, snapshotPolicy.GetShortDesc(ctx), userCred) - return snapshotPolicy, nil -} + syncOwnerId := provider.GetOwnerId() -func (spm *SSnapshotPolicyManager) FetchProjectId(ctx context.Context, userCred mcclient.TokenCredential, - syncOwnerId mcclient.IIdentityProvider, cloudSP cloudprovider.ICloudSnapshotPolicy, managerId string) string { - var newOwnerId mcclient.IIdentityProvider - if extProjectId := cloudSP.GetProjectId(); len(extProjectId) > 0 { - extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId) - if err != nil { - log.Errorln(err) - } else { - newOwnerId = extProject.GetOwnerId() - } + SyncCloudProject(ctx, userCred, self, syncOwnerId, ext, provider.Id) + if account, _ := provider.GetCloudaccount(); account != nil { + syncVirtualResourceMetadata(ctx, userCred, self, ext, account.ReadOnly) } - if newOwnerId == nil && syncOwnerId != nil && len(syncOwnerId.GetProjectId()) > 0 { - newOwnerId = syncOwnerId - } - if newOwnerId == nil { - newOwnerId = userCred - } - return newOwnerId.GetProjectId() -} -func (sp *SSnapshotPolicy) Key() uint64 { - var key uint64 - key |= (uint64(sp.RepeatWeekdays) << 56) | (uint64(sp.TimePoints) << 24) - // that sp.RetentionDays is -1 means permanent retention, sp.RetentionDays+1 must be less than 2^23 - r := sp.RetentionDays + 1&(1<<24-1) - key |= uint64(r) << 1 - if sp.IsActivated.IsTrue() { - key |= 1 + err = self.SyncDisks(ctx, userCred, ext) + if err != nil { + return errors.Wrapf(err, "SyncDisks") } - return key + + return nil } -func (sp *SSnapshotPolicy) Equals(cloudSP cloudprovider.ICloudSnapshotPolicy) bool { - rws, err := cloudSP.GetRepeatWeekdays() +func (self *SCloudregion) newFromCloudPolicy( + ctx context.Context, userCred mcclient.TokenCredential, + provider *SCloudprovider, + ext cloudprovider.ICloudSnapshotPolicy, +) (*SSnapshotPolicy, error) { + policy := &SSnapshotPolicy{} + policy.SetModelManager(SnapshotPolicyManager, policy) + policy.CloudregionId = self.Id + policy.ManagerId = provider.Id + policy.RetentionDays = ext.GetRetentionDays() + var err error + policy.RepeatWeekdays, err = ext.GetRepeatWeekdays() if err != nil { - return false + return nil, errors.Wrapf(err, "GetRepeatWeekdays") } - tps, err := cloudSP.GetTimePoints() + policy.TimePoints, err = ext.GetTimePoints() if err != nil { - return false + return nil, errors.Wrapf(err, "GetTimePoints") } - repeatWeekdays := SnapshotPolicyManager.RepeatWeekdaysParseIntArray(rws) - timePoints := SnapshotPolicyManager.TimePointsParseIntArray(tps) + policy.Status = ext.GetStatus() + policy.Name = ext.GetName() + syncOwnerId := provider.GetOwnerId() - return sp.RetentionDays == cloudSP.GetRetentionDays() && sp.RepeatWeekdays == repeatWeekdays && sp. - TimePoints == timePoints && sp.IsActivated.Bool() == cloudSP.IsActivated() -} + err = func() error { + lockman.LockRawObject(ctx, SnapshotPolicyManager.Keyword(), "name") + defer lockman.ReleaseRawObject(ctx, SnapshotPolicyManager.Keyword(), "name") + + newName, err := db.GenerateName(ctx, SnapshotPolicyManager, syncOwnerId, policy.Name) + if err != nil { + return err + } + policy.Name = newName -func (manager *SSnapshotPolicyManager) getProviderSnapshotPolicies(region *SCloudregion, provider *SCloudprovider) ([]SSnapshotPolicy, error) { - if region == nil && provider == nil { - return nil, fmt.Errorf("Region is nil or provider is nil") + return SnapshotPolicyManager.TableSpec().Insert(ctx, policy) + }() + if err != nil { + return nil, errors.Wrapf(err, "Insert") } - snapshotPolicies := make([]SSnapshotPolicy, 0) - q := manager.Query().Equals("cloudregion_id", region.Id).Equals("manager_id", provider.Id) - err := db.FetchModelObjects(manager, q, &snapshotPolicies) + SyncCloudProject(ctx, userCred, policy, syncOwnerId, ext, provider.Id) + syncVirtualResourceMetadata(ctx, userCred, policy, ext, false) + + err = policy.SyncDisks(ctx, userCred, ext) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "SyncDisks") } - return snapshotPolicies, nil + + return policy, nil } func (sp *SSnapshotPolicy) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { @@ -668,219 +391,109 @@ func (sp *SSnapshotPolicy) Delete(ctx context.Context, userCred mcclient.TokenCr } func (sp *SSnapshotPolicy) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + err := SnapshotPolicyDiskManager.RemoveBySnapshotpolicy(sp.Id) + if err != nil { + return errors.Wrapf(err, "delete snapshot policy disks for policy %s", sp.Name) + } return db.DeleteModel(ctx, userCred, sp) } -func (self *SSnapshotPolicyManager) RepeatWeekdaysParseIntArray(nums []int) uint8 { - return uint8(bitmap.IntArray2Uint(nums)) -} - -func (self *SSnapshotPolicyManager) RepeatWeekdaysToIntArray(n uint8) []int { - return bitmap.Uint2IntArray(uint32(n)) -} - -func (self *SSnapshotPolicyManager) TimePointsParseIntArray(nums []int) uint32 { - return bitmap.IntArray2Uint(nums) -} - -func (self *SSnapshotPolicyManager) TimePointsToIntArray(n uint32) []int { - return bitmap.Uint2IntArray(n) -} - -func computeNextSyncTime(weekDays, timePoints []int, base time.Time) time.Time { - if base.IsZero() { - base = time.Now() - } - - // Add 1 hour to base to prevent the calculation result from being equal to the input base - base = base.Add(time.Hour) - base = base.Truncate(time.Hour) - - baseWeekday := int(base.Weekday()) - if baseWeekday == 0 { - baseWeekday = 7 +func (sp *SSnapshotPolicy) StartBindDisksTask(ctx context.Context, userCred mcclient.TokenCredential, diskIds []string) error { + sp.SetStatus(userCred, api.SNAPSHOT_POLICY_APPLY, jsonutils.Marshal(diskIds).String()) + params := jsonutils.Marshal(map[string]interface{}{"disk_ids": diskIds}).(*jsonutils.JSONDict) + task, err := taskman.TaskManager.NewTask(ctx, "SnapshotpolicyBindDisksTask", sp, userCred, params, "", "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") } - weekDays = append(weekDays, weekDays[0]+7) - indexw := sort.SearchInts(weekDays, baseWeekday) - addDay := weekDays[indexw] - baseWeekday - nextTime := base.AddDate(0, 0, addDay) + return task.ScheduleRun(nil) +} - // find timePoint closest to the base - var newHour int - if addDay > 0 { - newHour = timePoints[0] - } else { - baseHour := base.Hour() - index := sort.SearchInts(timePoints, baseHour) - if index == len(timePoints) { - // indexw + 1 must less than len(weekDays) - addDay = weekDays[indexw+1] - baseWeekday - nextTime = base.AddDate(0, 0, addDay) +func (sp *SSnapshotPolicy) PerformBindDisks( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.SnapshotPolicyDisksInput, +) (jsonutils.JSONObject, error) { + if len(input.Disks) == 0 { + return nil, httperrors.NewMissingParameterError("disks") + } + diskIds := []string{} + for i := range input.Disks { + diskObj, err := validators.ValidateModel(userCred, DiskManager, &input.Disks[i]) + if err != nil { + return nil, err } - index = index % len(timePoints) - if timePoints[index] == baseHour { - index = (index + 1) % len(timePoints) - newHour = timePoints[index] - } else { - newHour = timePoints[index] + disk := diskObj.(*SDisk) + if len(sp.ManagerId) > 0 { + storage, err := disk.GetStorage() + if err != nil { + return nil, errors.Wrapf(err, "GetStorage for disk %s", disk.Name) + } + if storage.ManagerId != sp.ManagerId { + return nil, httperrors.NewConflictError("The snapshot policy %s and disk account are different", sp.Name) + } + zone, err := storage.GetZone() + if err != nil { + return nil, errors.Wrapf(err, "GetZone") + } + if sp.CloudregionId != zone.CloudregionId { + return nil, httperrors.NewConflictError("The snapshot policy %s and the disk are in different region", sp.Name) + } } - } - nextTime = time.Date(nextTime.Year(), nextTime.Month(), nextTime.Day(), newHour, 0, 0, 0, base.Location()) - - return nextTime -} - -func (sp *SSnapshotPolicy) ComputeNextSyncTime(base time.Time) time.Time { - weekDays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays) - timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints) - if sp.RetentionDays <= 0 { - return computeNextSyncTime(weekDays, timePoints, base) - } - // A snapshotpolicy takes effect every Monday with keeping snapshot 3 days. So, the snapshots should be synchronized every Monday (snapshots) and Thursdays (release snapshots). - set := sets.NewInt(weekDays...) - for _, day := range weekDays { - newDay := (day + sp.RetentionDays) % 7 - if newDay == 0 { - newDay = 7 + if !utils.IsInStringArray(disk.Id, diskIds) { + diskIds = append(diskIds, disk.Id) } - set.Insert(newDay) } - return computeNextSyncTime(set.List(), timePoints, base) + return nil, sp.StartBindDisksTask(ctx, userCred, diskIds) } -func (sp *SSnapshotPolicy) GenerateCreateSpParams() *cloudprovider.SnapshotPolicyInput { - intWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays) - intTimePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints) - - return &cloudprovider.SnapshotPolicyInput{ - RetentionDays: sp.RetentionDays, - RepeatWeekdays: intWeekdays, - TimePoints: intTimePoints, - PolicyName: sp.Name, - } -} - -// ==================================================== action ========================================================= - -func (sp *SSnapshotPolicy) PerformCache(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - - regionId := jsonutils.GetAnyString(data, []string{"region_id", "cloudregion_id"}) - if len(regionId) == 0 { - return nil, httperrors.NewMissingParameterError("region_id or cloudregion_id") - } - providerId, err := data.GetString("provider_id") - if err != nil { - return nil, httperrors.NewMissingParameterError("provider_id") - } - _, err = SnapshotPolicyCacheManager.NewCache(ctx, userCred, sp.Id, regionId, providerId) +func (sp *SSnapshotPolicy) StartUnbindDisksTask(ctx context.Context, userCred mcclient.TokenCredential, diskIds []string) error { + sp.SetStatus(userCred, api.SNAPSHOT_POLICY_CANCEL, jsonutils.Marshal(diskIds).String()) + params := jsonutils.Marshal(map[string]interface{}{"disk_ids": diskIds}).(*jsonutils.JSONDict) + task, err := taskman.TaskManager.NewTask(ctx, "SnapshotpolicyUnbindDisksTask", sp, userCred, params, "", "", nil) if err != nil { - return nil, err + return errors.Wrapf(err, "NewTask") } - - return nil, nil + return task.ScheduleRun(nil) } -func (sp *SSnapshotPolicy) PerformBindDisks(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - - disks := jsonutils.GetArrayOfPrefix(data, "disk") - if len(disks) == 0 { - return nil, httperrors.NewMissingParameterError("disk.0 disk.1 ... ") - } - - //database - diskSlice := make([]*SDisk, len(disks)) - for i := range disks { - diskId, _ := disks[i].GetString() - disk := DiskManager.FetchDiskById(diskId) - if disk == nil { - return nil, httperrors.NewInputParameterError("no such disk %s", diskId) - } - disk.SetModelManager(DiskManager, disk) - diskSlice[i] = disk - } - - taskDisk := make([]*SDisk, 0, len(diskSlice)) - taskSpd := make([]*SSnapshotPolicyDisk, 0, len(diskSlice)) - for _, disk := range diskSlice { - spd, err := SnapshotPolicyDiskManager.newSnapshotpolicyDisk(ctx, userCred, sp, disk) - if err == ErrExistSD { - if spd.Status == "init" { - taskDisk = append(taskDisk, disk) - taskSpd = append(taskSpd, spd) - } - continue - } +func (sp *SSnapshotPolicy) PerformUnbindDisks( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.SnapshotPolicyDisksInput, +) (jsonutils.JSONObject, error) { + if len(input.Disks) == 0 { + return nil, httperrors.NewMissingParameterError("disks") + } + diskIds := []string{} + for i := range input.Disks { + diskObj, err := validators.ValidateModel(userCred, DiskManager, &input.Disks[i]) if err != nil { - return nil, fmt.Errorf("oper for database error") + return nil, err } - taskDisk = append(taskDisk, disk) - taskSpd = append(taskSpd, spd) - } - - for i := range taskDisk { - // field 'need_detach' is not needed, because the the subject is snapshot policy not disk - taskdata := jsonutils.NewDict() - taskdata.Add(jsonutils.Marshal(taskSpd[i]), "snapshotPolicyDisk") - taskdata.Add(jsonutils.Marshal(sp), "snapshotPolicy") - if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyApplyTask", taskDisk[i], userCred, nil, - "", "", nil); err != nil { - continue - } else { - task.ScheduleRun(taskdata) + disk := diskObj.(*SDisk) + if utils.IsInStringArray(disk.Id, diskIds) { + diskIds = append(diskIds, disk.Id) } } - - return nil, nil + return nil, sp.StartUnbindDisksTask(ctx, userCred, diskIds) } -func (sp *SSnapshotPolicy) PerformUnbindDisks(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - - disks := jsonutils.GetArrayOfPrefix(data, "disk") - if len(disks) == 0 { - return nil, httperrors.NewMissingParameterError("disk.0 disk.1 ... ") - } - - diskSlice := make([]*SDisk, len(disks)) - for i := range disks { - diskId, _ := disks[i].GetString() - disk := DiskManager.FetchDiskById(diskId) - if disk == nil { - return nil, httperrors.NewInputParameterError("no such disk %s", diskId) - } - disk.SetModelManager(DiskManager, disk) - diskSlice[i] = disk - } - - taskDisk := make([]*SDisk, 0, len(diskSlice)) - taskSpd := make([]*SSnapshotPolicyDisk, 0, len(diskSlice)) - for _, disk := range diskSlice { - spd, err := SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(sp.Id, disk.GetId()) - if err != nil { - continue - } - if spd == nil { - continue - } - taskSpd = append(taskSpd, spd) - taskDisk = append(taskDisk, disk) +func (self *SSnapshotPolicy) PerformSyncstatus( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + if self.CloudregionId == api.DEFAULT_REGION_ID { + return nil, self.SetStatus(userCred, apis.STATUS_AVAILABLE, "") } + return nil, self.StartSyncstatusTask(ctx, userCred, "") +} - for i := range taskDisk { - taskdata := jsonutils.NewDict() - taskdata.Add(jsonutils.NewString(sp.Id), "snapshot_policy_id") - taskdata.Add(jsonutils.Marshal(taskSpd[i]), "snapshotPolicyDisk") - taskSpd[i].SetStatus(userCred, api.SNAPSHOT_POLICY_DISK_DELETING, "") - if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCancelTask", taskDisk[i], userCred, nil, "", "", - nil); err != nil { - continue - } else { - task.ScheduleRun(taskdata) - } - } - return nil, nil +func (sp *SSnapshotPolicy) StartSyncstatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + return StartResourceSyncStatusTask(ctx, userCred, sp, "SnapshotpolicySyncstatusTask", parentTaskId) } // 快照策略列表 @@ -894,13 +507,21 @@ func (manager *SSnapshotPolicyManager) ListItemFilter( if err != nil { return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter") } - if input.IsActivated != nil { - if *input.IsActivated { - q = q.IsTrue("is_activated") - } else { - q = q.IsFalse("is_activated") - } + q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, input.ExternalizedResourceBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter") + } + + q, err = manager.SManagedResourceBaseManager.ListItemFilter(ctx, q, userCred, input.ManagedResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemFilter") + } + + q, err = manager.SCloudregionResourceBaseManager.ListItemFilter(ctx, q, userCred, input.RegionalFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemFilter") } + return q, nil } @@ -916,6 +537,16 @@ func (manager *SSnapshotPolicyManager) OrderByExtraFields( if err != nil { return nil, errors.Wrap(err, "SVirtualResourceBaseManager.OrderByExtraFields") } + + q, err = manager.SManagedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, input.ManagedResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.OrderByExtraFields") + } + q, err = manager.SCloudregionResourceBaseManager.OrderByExtraFields(ctx, q, userCred, input.RegionalFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.OrderByExtraFields") + } + if db.NeedOrderQuery([]string{input.OrderByBindDiskCount}) { sdQ := SnapshotPolicyDiskManager.Query() sdSQ := sdQ.AppendField(sdQ.Field("snapshotpolicy_id"), sqlchemy.COUNT("disk_count")).GroupBy("snapshotpolicy_id").SubQuery() @@ -935,5 +566,198 @@ func (manager *SSnapshotPolicyManager) QueryDistinctExtraField(q *sqlchemy.SQuer return q, nil } + q, err = manager.SManagedResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + q, err = manager.SCloudregionResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + return q, httperrors.ErrNotFound } + +func (manager *SSnapshotPolicyManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemExportKeys") + } + + if keys.ContainsAny(manager.SCloudregionResourceBaseManager.GetExportKeys()...) { + q, err = manager.SCloudregionResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemExportKeys") + } + } + if keys.ContainsAny(manager.SManagedResourceBaseManager.GetExportKeys()...) { + q, err = manager.SManagedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemExportKeys") + } + } + + return q, nil +} + +func (self *SSnapshotPolicy) GetISnapshotPolicy(ctx context.Context) (cloudprovider.ICloudSnapshotPolicy, error) { + if len(self.ExternalId) == 0 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "empty external id") + } + iRegion, err := self.GetIRegion(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetIRegion") + } + return iRegion.GetISnapshotPolicyById(self.ExternalId) +} + +func (self *SSnapshotPolicy) GetIRegion(ctx context.Context) (cloudprovider.ICloudRegion, error) { + region, err := self.GetRegion() + if err != nil { + return nil, errors.Wrapf(err, "GetRegion") + } + provider, err := self.GetProvider(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetProvider") + } + return provider.GetIRegionById(region.ExternalId) +} + +func (self *SSnapshotPolicy) GetCloudprovider() (*SCloudprovider, error) { + providerObj, err := CloudproviderManager.FetchById(self.ManagerId) + if err != nil { + return nil, errors.Wrapf(err, "FetchById") + } + return providerObj.(*SCloudprovider), nil +} + +func (self *SSnapshotPolicy) GetProvider(ctx context.Context) (cloudprovider.ICloudProvider, error) { + manager, err := self.GetCloudprovider() + if err != nil { + return nil, errors.Wrapf(err, "GetProvider") + } + return manager.GetProvider(ctx) +} + +func (self *SSnapshotPolicy) GetUnbindDisks(diskIds []string) ([]SDisk, error) { + sq := SnapshotPolicyDiskManager.Query("disk_id").Equals("snapshotpolicy_id", self.Id).SubQuery() + q := DiskManager.Query().In("id", diskIds) + q = q.Filter(sqlchemy.NotIn(q.Field("id"), sq)) + ret := []SDisk{} + err := db.FetchModelObjects(DiskManager, q, &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (self *SSnapshotPolicy) GetBindDisks(diskIds []string) ([]SDisk, error) { + sq := SnapshotPolicyDiskManager.Query("disk_id").Equals("snapshotpolicy_id", self.Id).SubQuery() + q := DiskManager.Query().In("id", diskIds) + q = q.Filter(sqlchemy.In(q.Field("id"), sq)) + ret := []SDisk{} + err := db.FetchModelObjects(DiskManager, q, &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (self *SSnapshotPolicy) GetDisks() ([]SDisk, error) { + sq := SnapshotPolicyDiskManager.Query().Equals("snapshotpolicy_id", self.Id).SubQuery() + q := DiskManager.Query() + q = q.Join(sq, sqlchemy.Equals(q.Field("id"), sq.Field("disk_id"))) + ret := []SDisk{} + err := db.FetchModelObjects(DiskManager, q, &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (sp *SSnapshotPolicy) BindDisks(ctx context.Context, disks []SDisk) error { + for i := range disks { + spd := &SSnapshotPolicyDisk{} + spd.SetModelManager(SnapshotPolicyDiskManager, spd) + spd.DiskId = disks[i].Id + spd.SnapshotpolicyId = sp.Id + err := SnapshotPolicyDiskManager.TableSpec().Insert(ctx, spd) + if err != nil { + return err + } + } + return nil +} + +func (sp *SSnapshotPolicy) UnbindDisks(diskIds []string) error { + vars := []interface{}{sp.Id} + placeholders := make([]string, len(diskIds)) + for i := range placeholders { + placeholders[i] = "?" + vars = append(vars, diskIds[i]) + } + _, err := sqlchemy.GetDB().Exec( + fmt.Sprintf( + "delete from %s where snapshotpolicy_id = ? and disk_id in (%s)", + SnapshotPolicyDiskManager.TableSpec().Name(), strings.Join(placeholders, ","), + ), vars..., + ) + return err +} + +func (sp *SSnapshotPolicy) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudSnapshotPolicy) error { + extIds, err := ext.GetApplyDiskIds() + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotImplemented || errors.Cause(err) == cloudprovider.ErrNotSupported { + return nil + } + return errors.Wrapf(err, "GetApplyDiskIds") + } + { + sq := SnapshotPolicyDiskManager.Query("disk_id").Equals("snapshotpolicy_id", sp.Id).SubQuery() + q := DiskManager.Query().In("id", sq).NotIn("external_id", extIds) + needCancel := []SDisk{} + err = db.FetchModelObjects(DiskManager, q, &needCancel) + if err != nil { + return errors.Wrapf(err, "db.FetchModelObjects") + } + diskIds := []string{} + for _, disk := range needCancel { + diskIds = append(diskIds, disk.Id) + } + if len(diskIds) > 0 { + err = sp.UnbindDisks(diskIds) + if err != nil { + return errors.Wrapf(err, "UnbindDisks") + } + } + } + { + sq := SnapshotPolicyDiskManager.Query("disk_id").Equals("snapshotpolicy_id", sp.Id).SubQuery() + storages := StorageManager.Query().Equals("manager_id", sp.ManagerId).SubQuery() + q := DiskManager.Query() + q = q.Join(storages, sqlchemy.Equals(q.Field("storage_id"), storages.Field("id"))) + q = q.Filter( + sqlchemy.AND( + sqlchemy.NotIn(q.Field("id"), sq), + sqlchemy.In(q.Field("external_id"), extIds), + ), + ) + needApply := []SDisk{} + err = db.FetchModelObjects(DiskManager, q, &needApply) + if err != nil { + return errors.Wrapf(err, "db.FetchModelObjects") + } + err = sp.BindDisks(ctx, needApply) + if err != nil { + return errors.Wrapf(err, "BindDisks") + } + } + return nil +} diff --git a/pkg/compute/models/snapshotpolicy_test.go b/pkg/compute/models/snapshotpolicy_test.go deleted file mode 100644 index 704ff556089..00000000000 --- a/pkg/compute/models/snapshotpolicy_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "testing" - "time" - - "yunion.io/x/log" - "yunion.io/x/pkg/tristate" -) - -func TestSSnapshotPolicy_Key(t *testing.T) { - cases := []struct { - in *SSnapshotPolicy - want uint64 - }{ - { - &SSnapshotPolicy{ - RepeatWeekdays: 0, - TimePoints: 0, - RetentionDays: 7, - IsActivated: tristate.True, - }, - 15 + 2, - }, - { - &SSnapshotPolicy{ - RepeatWeekdays: 11, - }, - 11<<56 + 2, - }, - { - &SSnapshotPolicy{ - TimePoints: 234, - }, - 234<<24 + 2, - }, - { - &SSnapshotPolicy{ - RetentionDays: -1, - }, - 0, - }, - { - &SSnapshotPolicy{ - RepeatWeekdays: 13, - TimePoints: 123, - RetentionDays: 7, - }, - 936748724556660752, - }, - } - - for i, c := range cases { - g := c.in.Key() - if c.want != g { - log.Errorf("the %d case, want %d, get %d", i, c.want, g) - } - } -} - -func TestSSnapshotPolicy_ComputeNextSyncTime(t *testing.T) { - timeStr := "2006-01-02 15:04:05" - t.Run("base test", func(t *testing.T) { - cases := []struct { - in *SSnapshotPolicy - base string - want string - }{ - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{2}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{4}), - }, - base: "2020-10-31 00:00:00", - want: "2020-11-03 04:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5, 7}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{2, 6}), - }, - base: "2020-10-31 00:00:00", - want: "2020-11-01 02:00:00", - }, - } - for _, c := range cases { - base, _ := time.Parse(timeStr, c.base) - want, _ := time.Parse(timeStr, c.want) - real := c.in.ComputeNextSyncTime(base) - if want != real { - t.Fatalf("want: %s, real: %s", want, real) - } - } - }) - t.Run("same day", func(t *testing.T) { - cases := []struct { - in *SSnapshotPolicy - base string - want string - }{ - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{2}), - }, - base: "2020-10-31 00:00:00", - want: "2020-10-31 02:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{2}), - }, - base: "2020-10-31 02:00:00", - want: "2020-11-01 02:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{2}), - }, - base: "2020-10-31 01:00:00", - want: "2020-10-31 02:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{2, 4}), - }, - base: "2020-10-31 05:00:00", - want: "2020-11-07 02:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{1, 2, 3, 4, 5, 6, 7}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{0}), - }, - base: "2020-10-31 05:00:00", - want: "2020-11-01 00:00:00", - }, - } - for _, c := range cases { - base, _ := time.Parse(timeStr, c.base) - want, _ := time.Parse(timeStr, c.want) - real := c.in.ComputeNextSyncTime(base) - if want != real { - t.Fatalf("want: %s, real: %s", want, real) - } - } - }) - t.Run("retentionday", func(t *testing.T) { - cases := []struct { - in *SSnapshotPolicy - base string - want string - }{ - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{4}), - RetentionDays: 2, - }, - base: "2020-10-31 04:00:00", - want: "2020-11-01 04:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{4}), - RetentionDays: 8, - }, - base: "2020-10-31 04:00:00", - want: "2020-11-01 04:00:00", - }, - { - in: &SSnapshotPolicy{ - RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{1, 6}), - TimePoints: SnapshotPolicyManager.TimePointsParseIntArray([]int{4}), - RetentionDays: 4, - }, - base: "2020-10-31 04:00:00", - want: "2020-11-02 04:00:00", - }, - } - for _, c := range cases { - base, _ := time.Parse(timeStr, c.base) - want, _ := time.Parse(timeStr, c.want) - real := c.in.ComputeNextSyncTime(base) - if want != real { - t.Fatalf("want: %s, real: %s", want, real) - } - } - }) -} diff --git a/pkg/compute/models/snapshotpolicycache.go b/pkg/compute/models/snapshotpolicycache.go deleted file mode 100644 index 106e0eb3207..00000000000 --- a/pkg/compute/models/snapshotpolicycache.go +++ /dev/null @@ -1,566 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "context" - "fmt" - "strings" - "time" - - "yunion.io/x/cloudmux/pkg/cloudprovider" - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/rbacscope" - "yunion.io/x/sqlchemy" - - api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/stringutils2" -) - -type SSnapshotPolicyCacheManager struct { - db.SStatusStandaloneResourceBaseManager - db.SExternalizedResourceBaseManager - SCloudregionResourceBaseManager - SManagedResourceBaseManager - SSnapshotPolicyResourceBaseManager -} - -type SSnapshotPolicyCache struct { - db.SStatusStandaloneResourceBase - db.SExternalizedResourceBase - SCloudregionResourceBase - SManagedResourceBase - SSnapshotPolicyResourceBase `width:"128" charset:"ascii" create:"required"` - // SnapshotpolicyId string `width:"128" charset:"ascii" create:"required"` -} - -var SnapshotPolicyCacheManager *SSnapshotPolicyCacheManager - -func init() { - SnapshotPolicyCacheManager = &SSnapshotPolicyCacheManager{ - SStatusStandaloneResourceBaseManager: db.NewStatusStandaloneResourceBaseManager( - SSnapshotPolicyCache{}, - "snapshotpolicycache_tbl", - "snapshotpolicycache", - "snapshotpolicycaches", - ), - } - SnapshotPolicyCacheManager.SetVirtualObject(SnapshotPolicyCacheManager) -} - -func NewSSnapshotPolicyCache(snapshotpolicyId, cloudregionId, externalId string) *SSnapshotPolicyCache { - cache := SSnapshotPolicyCache{ - // SnapshotpolicyId: snapshotpolicyId, - SCloudregionResourceBase: SCloudregionResourceBase{ - CloudregionId: cloudregionId, - }, - SExternalizedResourceBase: db.SExternalizedResourceBase{ - ExternalId: externalId, - }, - } - cache.SnapshotpolicyId = snapshotpolicyId - cache.SetModelManager(SnapshotPolicyCacheManager, &cache) - return &cache -} - -// 快照策略缓存列表 -func (spcm *SSnapshotPolicyCacheManager) ListItemFilter( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyCacheListInput, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = spcm.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StatusStandaloneResourceListInput) - if err != nil { - return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ListItemFilter") - } - - q, err = spcm.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput) - if err != nil { - return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter") - } - - q, err = spcm.SManagedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ManagedResourceListInput) - if err != nil { - return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemFilter") - } - - q, err = spcm.SCloudregionResourceBaseManager.ListItemFilter(ctx, q, userCred, query.RegionalFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemFilter") - } - - q, err = spcm.SSnapshotPolicyResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SnapshotPolicyFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SSnapshotPolicyResourceBaseManager.ListItemFilter") - } - /*if snapshotpolicyIden := query.Snapshotpolicy; len(snapshotpolicyIden) > 0 { - snapshotpolicy, err := SnapshotPolicyManager.FetchByIdOrName(userCred, snapshotpolicyIden) - if err != nil { - if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(SnapshotPolicyManager.Keyword(), snapshotpolicyIden) - } else { - return nil, err - } - } - q = q.Equals("snapshotpolicy_id", snapshotpolicy.GetId()) - }*/ - return q, nil -} - -func (spcm *SSnapshotPolicyCacheManager) OrderByExtraFields( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyCacheListInput, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = spcm.SStatusStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StatusStandaloneResourceListInput) - if err != nil { - return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.OrderByExtraFields") - } - q, err = spcm.SManagedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.ManagedResourceListInput) - if err != nil { - return nil, errors.Wrap(err, "SManagedResourceBaseManager.OrderByExtraFields") - } - q, err = spcm.SCloudregionResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.RegionalFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.OrderByExtraFields") - } - q, err = spcm.SSnapshotPolicyResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SnapshotPolicyFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SSnapshotPolicyResourceBaseManager.OrderByExtraFields") - } - - return q, nil -} - -func (spcm *SSnapshotPolicyCacheManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { - var err error - - q, err = spcm.SStatusStandaloneResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - q, err = spcm.SManagedResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - q, err = spcm.SCloudregionResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - q, err = spcm.SSnapshotPolicyResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - - return q, httperrors.ErrNotFound -} - -func (spc *SSnapshotPolicyCache) GetIRegion(ctx context.Context) (cloudprovider.ICloudRegion, error) { - provider, err := spc.GetDriver(ctx) - if err != nil { - return nil, err - } - if region := CloudregionManager.FetchRegionById(spc.CloudregionId); region != nil { - return provider.GetIRegionById(region.ExternalId) - } - return nil, fmt.Errorf("failed to find iregion for snapshotpolicycache %s: cloudregion %s manager %s", spc.Id, - spc.CloudregionId, spc.ManagerId) -} - -func (spc *SSnapshotPolicyCache) GetSnapshotPolicy() (*SSnapshotPolicy, error) { - model, err := SnapshotPolicyManager.FetchById(spc.SnapshotpolicyId) - if err != nil { - return nil, fmt.Errorf("failed to fetchsnapshotpolicy by %s", spc.SnapshotpolicyId) - } - return model.(*SSnapshotPolicy), nil -} - -func (spcm *SSnapshotPolicyCacheManager) FetchCustomizeColumns( - ctx context.Context, - userCred mcclient.TokenCredential, - query jsonutils.JSONObject, - objs []interface{}, - fields stringutils2.SSortedStrings, - isList bool, -) []api.SnapshotPolicyCacheDetails { - rows := make([]api.SnapshotPolicyCacheDetails, len(objs)) - - stdRows := spcm.SStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - manRows := spcm.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - regionRows := spcm.SCloudregionResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - snapshotPolicyRows := spcm.SSnapshotPolicyResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - for i := range rows { - rows[i] = api.SnapshotPolicyCacheDetails{ - StatusStandaloneResourceDetails: stdRows[i], - ManagedResourceInfo: manRows[i], - CloudregionResourceInfo: regionRows[i], - SnapshotPolicyResourceInfo: snapshotPolicyRows[i], - } - } - - return rows -} - -// =============================================== detach and delete =================================================== - -func (spc *SSnapshotPolicyCache) RealDetele(ctx context.Context, userCred mcclient.TokenCredential) error { - return db.DeleteModel(ctx, userCred, spc) -} - -func (spc *SSnapshotPolicyCache) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { - return nil -} - -func (spc *SSnapshotPolicyCache) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, - query jsonutils.JSONObject, data jsonutils.JSONObject) error { - spc.SetStatus(userCred, api.SNAPSHOT_POLICY_CACHE_STATUS_DELETING, "") - task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCacheDeleteTask", spc, userCred, nil, "", "", nil) - if err != nil { - return err - } - task.ScheduleRun(nil) - return nil -} - -// ================================================= new and regist ==================================================== - -func (spcm *SSnapshotPolicyCacheManager) NewCache(ctx context.Context, userCred mcclient.TokenCredential, - snapshotPolicyId, regionId, providerId string) (*SSnapshotPolicyCache, error) { - - snapshotPolicyCache := NewSSnapshotPolicyCache(snapshotPolicyId, regionId, "") - snapshotPolicyCache.ManagerId = providerId - - err := snapshotPolicyCache.CreateCloudSnapshotPolicy(ctx) - if err != nil { - return nil, err - } - snapshotPolicyCache.Status = api.SNAPSHOT_POLICY_CACHE_STATUS_READY - - // should have lock - if err := spcm.TableSpec().Insert(ctx, snapshotPolicyCache); err != nil { - return nil, errors.Wrapf(err, "insert snapshotpolicycache failed") - } - return snapshotPolicyCache, nil -} - -func (spcm *SSnapshotPolicyCacheManager) NewCacheWithExternalId(ctx context.Context, userCred mcclient.TokenCredential, - snapshotPolicyId, externalId, regionId, providerId string, name string) (*SSnapshotPolicyCache, error) { - - snapshotPolicyCache := NewSSnapshotPolicyCache(snapshotPolicyId, regionId, externalId) - snapshotPolicyCache.ManagerId = providerId - - snapshotPolicyCache.Status = api.SNAPSHOT_POLICY_CACHE_STATUS_READY - snapshotPolicyCache.Name = name - // should have lock - if err := spcm.TableSpec().Insert(ctx, snapshotPolicyCache); err != nil { - return nil, errors.Wrapf(err, "insert snapshotpolicycache failed") - } - return snapshotPolicyCache, nil -} - -func (spcm *SSnapshotPolicyCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, snapshotPolicyId, - regionId string, providerId string) (*SSnapshotPolicyCache, error) { - - // Many request about same snapshot policy with same region and provider coming will cause many same cache - // building without lock, so we must Lock. - lockman.LockRawObject(ctx, snapshotPolicyId, regionId+providerId) - defer lockman.ReleaseRawObject(ctx, snapshotPolicyId, regionId+providerId) - snapshotPolicyCache, err := spcm.FetchSnapshotPolicyCache(snapshotPolicyId, regionId, providerId) - // error - if err != nil { - return nil, err - } - - // no cache - if snapshotPolicyCache != nil { - return snapshotPolicyCache, nil - } - - return spcm.NewCache(ctx, userCred, snapshotPolicyId, regionId, providerId) -} - -// ==================================================== fetch ========================================================= - -func (spcm *SSnapshotPolicyCacheManager) FetchSnapshotpolicyCaheById(cacheId string) (*SSnapshotPolicyCache, error) { - q := spcm.FilterById(spcm.Query(), cacheId) - return spcm.fetchByQuery(q) -} - -func (spcm *SSnapshotPolicyCacheManager) FetchSnapshotPolicyCache(snapshotPolicyId, regionId, providerId string) (*SSnapshotPolicyCache, error) { - - q := spcm.Query() - q.Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("snapshotpolicy_id"), snapshotPolicyId), - sqlchemy.Equals(q.Field("cloudregion_id"), regionId), - sqlchemy.Equals(q.Field("manager_id"), providerId))) - - return spcm.fetchByQuery(q) -} - -func (spcm *SSnapshotPolicyCacheManager) FetchSnapshotPolicyCacheByExtId(externalId, regionId, - providerId string) (*SSnapshotPolicyCache, error) { - - q := spcm.Query() - q.Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("external_id"), externalId), - sqlchemy.Equals(q.Field("cloudregion_id"), regionId), - sqlchemy.Equals(q.Field("manager_id"), providerId))) - - return spcm.fetchByQuery(q) -} - -func (spcm *SSnapshotPolicyCacheManager) FetchAllByExtIds(externalIds []string, regionId, - providerId string) ([]SSnapshotPolicyCache, error) { - - q := spcm.Query().In("external_id", externalIds).Equals("cloudregion_id", regionId).Equals("manager_id", providerId) - return spcm.fetchAllByQuery(q) -} - -func (spcm *SSnapshotPolicyCacheManager) FetchAllBySnpId(snapshotPolicyId string) ([]SSnapshotPolicyCache, error) { - - q := spcm.Query().Equals("snapshotpolicy_id", snapshotPolicyId) - return spcm.fetchAllByQuery(q) -} - -func (spcm *SSnapshotPolicyCacheManager) FetchAllByRegionProvider(cloudregionId, - managerId string) ([]SSnapshotPolicyCache, error) { - - q := spcm.Query().Equals("cloudregion_id", cloudregionId).Equals("manager_id", managerId) - return spcm.fetchAllByQuery(q) - -} - -func (spcm *SSnapshotPolicyCacheManager) fetchAllByQuery(q *sqlchemy.SQuery) ([]SSnapshotPolicyCache, error) { - caches := make([]SSnapshotPolicyCache, 0, 1) - if err := db.FetchModelObjects(spcm, q, &caches); err != nil { - return nil, err - } - return caches, nil -} - -func (spcm *SSnapshotPolicyCacheManager) fetchByQuery(q *sqlchemy.SQuery) (*SSnapshotPolicyCache, error) { - count, err := q.CountWithError() - if err != nil { - return nil, err - } - if count == 0 { - return nil, nil - } - snapshotPolicyCache := SSnapshotPolicyCache{} - // if exist, only one - q.First(&snapshotPolicyCache) - snapshotPolicyCache.SetModelManager(spcm, &snapshotPolicyCache) - return &snapshotPolicyCache, nil -} - -// ============================================== cloud operation ====================================================== -// This function should call in the task. - -type sOperaResult struct { - err error - snapshotPolicyId string -} - -func (spcm *SSnapshotPolicyCacheManager) UpdateCloudSnapshotPolicy(snapshotPolicyId string, - input *cloudprovider.SnapshotPolicyInput) error { - //todo maybe - - return fmt.Errorf("Not implement") -} - -type snapshotPolicyTask struct { - ctx context.Context - userCred mcclient.TokenCredential - spc SSnapshotPolicyCache - retChan chan sOperaResult -} - -func (t *snapshotPolicyTask) Run() { - err := t.spc.DeleteCloudSnapshotPolicy(t.ctx) - if err != nil { - t.retChan <- sOperaResult{err, t.spc.GetId()} - return - } - err = t.spc.RealDetele(t.ctx, t.userCred) - if err != nil { - t.retChan <- sOperaResult{errors.Wrap(err, "delete cache in database failed"), t.spc.GetId()} - return - } - t.retChan <- sOperaResult{nil, t.spc.GetId()} - -} - -func (t *snapshotPolicyTask) Dump() string { - return "" -} - -func (spcm *SSnapshotPolicyCacheManager) DeleteCloudSnapshotPolices(ctx context.Context, - userCred mcclient.TokenCredential, snapshotPolicyId string) error { - - spCaches, err := spcm.FetchAllBySnpId(snapshotPolicyId) - if err != nil { - return errors.Wrapf(err, "fetch all snapshotPolicyCaches ofsnapshotpolicy %s failed", snapshotPolicyId) - } - - if len(spCaches) == 0 { - return nil - } - - wm := appsrv.NewWorkerManager("delete-cloud-snapshotpolices", len(spCaches), 1, false) - - task := &snapshotPolicyTask{ - ctx: ctx, - userCred: userCred, - retChan: make(chan sOperaResult), - } - - for i := range spCaches { - task.spc = spCaches[i] - wm.Run(task, nil, func(e error) { - task.retChan <- sOperaResult{e, task.spc.GetId()} - }) - } - - failedRecord := make([]string, 0) - for i := 0; i < len(spCaches); i++ { - ret := <-task.retChan - if ret.err != nil { - failedRecord = append(failedRecord, fmt.Sprintf("%s failed because that %s", ret.snapshotPolicyId, - ret.err.Error())) - } - } - - if len(failedRecord) != 0 { - return fmt.Errorf("delete: " + strings.Join(failedRecord, "; ")) - } - return nil -} - -func (spc *SSnapshotPolicyCache) CreateCloudSnapshotPolicy(ctx context.Context) error { - // create correspondingsnapshotpolicy in cloud - iregion, err := spc.GetIRegion(ctx) - if err != nil { - return err - } - snapshotPolicy, err := spc.GetSnapshotPolicy() - if err != nil { - return err - } - - externalId, err := iregion.CreateSnapshotPolicy(snapshotPolicy.GenerateCreateSpParams()) - if err != nil { - return errors.Wrap(err, "createsnapshotpolicy failed") - } - spc.ExternalId = externalId - spc.Name = snapshotPolicy.Name - - iPolicy, err := iregion.GetISnapshotPolicyById(externalId) - if err != nil { - return err - } - err = cloudprovider.WaitStatus(iPolicy, api.SNAPSHOT_POLICY_READY, 10*time.Second, 300*time.Second) - if err != nil { - return err - } - return nil -} - -func (spc *SSnapshotPolicyCache) DeleteCloudSnapshotPolicy(ctx context.Context) error { - if len(spc.ExternalId) > 0 { - iregion, err := spc.GetIRegion(ctx) - if err != nil { - return err - } - cloudSp, err := iregion.GetISnapshotPolicyById(spc.ExternalId) - if errors.Cause(err) == cloudprovider.ErrNotFound { - return nil - } - if err != nil { - return errors.Wrap(err, "fetch snapshotpolicy from cloud before deleting failed") - } - if cloudSp == nil { - return nil - } - return iregion.DeleteSnapshotPolicy(spc.ExternalId) - } - return nil -} - -func (spc *SSnapshotPolicyCache) UpdateCloudSnapshotPolicy(ctx context.Context, input *cloudprovider.SnapshotPolicyInput) error { - iregion, err := spc.GetIRegion(ctx) - if err != nil { - return err - } - - err = iregion.UpdateSnapshotPolicy(input, spc.ExternalId) - if err != nil { - return errors.Wrap(err, "createsnapshotpolicy failed") - } - - return nil -} - -func (manager *SSnapshotPolicyCacheManager) ListItemExportKeys(ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - keys stringutils2.SSortedStrings, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = manager.SStatusStandaloneResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ListItemExportKeys") - } - - if keys.ContainsAny(manager.SCloudregionResourceBaseManager.GetExportKeys()...) { - q, err = manager.SCloudregionResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemExportKeys") - } - } - if keys.ContainsAny(manager.SManagedResourceBaseManager.GetExportKeys()...) { - q, err = manager.SManagedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemExportKeys") - } - } - - return q, nil -} - -func (manager *SSnapshotPolicyCacheManager) ResourceScope() rbacscope.TRbacScope { - return rbacscope.ScopeProject -} - -func (spc *SSnapshotPolicyCache) GetOwnerId() mcclient.IIdentityProvider { - p, err := spc.GetSnapshotPolicy() - if err != nil { - log.Errorf("unable to get snapshotpolicy of snapshotpolicycache %s: %v", spc.GetId(), err) - return nil - } - return p.GetOwnerId() -} diff --git a/pkg/compute/models/snapshotpolicydisks.go b/pkg/compute/models/snapshotpolicydisks.go index a83bc5c180b..261e969c1d2 100644 --- a/pkg/compute/models/snapshotpolicydisks.go +++ b/pkg/compute/models/snapshotpolicydisks.go @@ -15,31 +15,16 @@ package models import ( - "bytes" - "context" - "database/sql" "fmt" - "strings" - "time" - "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/sets" "yunion.io/x/sqlchemy" - api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/stringutils2" ) type SSnapshotPolicyDiskManager struct { db.SVirtualJointResourceBaseManager - SSnapshotPolicyResourceBaseManager SDiskResourceBaseManager } @@ -58,9 +43,9 @@ func init() { SnapshotPolicyDiskManager = &SSnapshotPolicyDiskManager{ SVirtualJointResourceBaseManager: db.NewVirtualJointResourceBaseManager( SSnapshotPolicyDisk{}, - "snapshotpolicydisks_tbl", - "snapshotpolicydisk", - "snapshotpolicydisks", + "snapshot_policy_disks_tbl", + "snapshot_policy_disk", + "snapshot_policy_disks", DiskManager, SnapshotPolicyManager, ), @@ -73,584 +58,42 @@ func init() { type SSnapshotPolicyDisk struct { db.SVirtualJointResourceBase - SSnapshotPolicyResourceBase `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` - SDiskResourceBase `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` - Status string `width:"36" charset:"ascii" nullable:"false" default:"init" list:"user" create:"optional"` - NextSyncTime time.Time + SnapshotpolicyId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` + SDiskResourceBase `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` } -func (sd *SSnapshotPolicyDisk) SetStatus(userCred mcclient.TokenCredential, status string, reason string) error { - if sd.Status == status { - return nil - } - oldStatus := sd.Status - _, err := db.Update(sd, func() error { - sd.Status = status - return nil - }) - if err != nil { - return err - } - if userCred != nil { - notes := fmt.Sprintf("%s=>%s", oldStatus, status) - if len(reason) > 0 { - notes = fmt.Sprintf("%s: %s", notes, reason) - } - db.OpsLog.LogEvent(sd, db.ACT_UPDATE_STATUS, notes, userCred) - } - return nil -} - -func (manager *SSnapshotPolicyDiskManager) FetchCustomizeColumns( - ctx context.Context, - userCred mcclient.TokenCredential, - query jsonutils.JSONObject, - objs []interface{}, - fields stringutils2.SSortedStrings, - isList bool, -) []api.SnapshotPolicyDiskDetails { - rows := make([]api.SnapshotPolicyDiskDetails, len(objs)) - - virtRows := manager.SVirtualJointResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - snapIds := make([]string, len(rows)) - diskIds := make([]string, len(rows)) - - for i := range rows { - rows[i] = api.SnapshotPolicyDiskDetails{ - VirtualJointResourceBaseDetails: virtRows[i], - } - snapIds[i] = objs[i].(*SSnapshotPolicyDisk).SnapshotpolicyId - diskIds[i] = objs[i].(*SSnapshotPolicyDisk).DiskId - } - - snapIdMaps, err := db.FetchIdNameMap2(SnapshotPolicyManager, snapIds) - if err != nil { - log.Errorf("FetchIdNameMap2 fail for snapshot Ids %s", err) - return rows - } - - disks := make(map[string]SDisk) - err = db.FetchStandaloneObjectsByIds(DiskManager, diskIds, &disks) - if err != nil { - log.Errorf("FetchStandaloneObjectsByIds for disks fail %s", err) - return rows - } - - for i := range rows { - if name, ok := snapIdMaps[snapIds[i]]; ok { - rows[i].Snapshotpolicy = name - } - if disk, ok := disks[diskIds[i]]; ok { - rows[i].Disk = api.DiskDetails{} - jsonutils.Update(&rows[i].Disk, disk) - } - } - - return rows -} - -// ==================================================== fetch ========================================================== - -func (m *SSnapshotPolicyDiskManager) FetchBySnapshotPolicyDisk(spId, diskId string) (*SSnapshotPolicyDisk, error) { - q := m.Query().Equals("snapshotpolicy_id", spId).Equals("disk_id", diskId) - ret := make([]SSnapshotPolicyDisk, 0, 1) - err := db.FetchModelObjects(m, q, &ret) - if err != nil { - return nil, err - } - if len(ret) == 0 { - return nil, nil - } - return &ret[0], nil -} - -func (sdm *SSnapshotPolicyDiskManager) InitializeData() error { - diskQ := DiskManager.Query("id").SubQuery() - sdQ := sdm.Query().NotIn("disk_id", diskQ) - - var sds []SSnapshotPolicyDisk - err := db.FetchModelObjects(sdm, sdQ, &sds) - if err != nil { - return errors.Wrap(err, "unable to FetchModelObjects") - } - for i := range sds { - sd := &sds[i] - _, err := db.Update(sd, func() error { - return sd.MarkDelete() - }) - if err != nil { - return err - } - } - - sds = make([]SSnapshotPolicyDisk, 0) - q := sdm.Query().IsNullOrEmpty("next_sync_time") - err = db.FetchModelObjects(sdm, q, &sds) - if err != nil { - return err - } - - // fetch all snapshotpolicy - spIdSet := sets.NewString() - for i := range sds { - spIdSet.Insert(sds[i].SnapshotpolicyId) - } - sps, err := SnapshotPolicyManager.FetchAllByIds(spIdSet.UnsortedList()) - if err != nil { - return errors.Wrap(err, "FetchAllByIds") - } - spMap := make(map[string]*SSnapshotPolicy, len(sps)) - for i := range sps { - spMap[sps[i].GetId()] = &sps[i] - } - now := time.Now() - for i := range sds { - sd := &sds[i] - _, err := db.Update(sd, func() error { - sd.NextSyncTime = spMap[sd.SnapshotpolicyId].ComputeNextSyncTime(now) - return nil - }) - if err != nil { - return errors.Wrap(err, "db.Update") - } - } - return nil -} - -func (m *SSnapshotPolicyDiskManager) FetchAllByDiskID(ctx context.Context, userCred mcclient.TokenCredential, - diskID string) ([]SSnapshotPolicyDisk, error) { - - return m.fetchAll(ctx, userCred, m.GetMasterFieldName(), diskID) -} - -func (m *SSnapshotPolicyDiskManager) FetchAllBySnapshotpolicyID(ctx context.Context, userCred mcclient.TokenCredential, - snapshotPolicyID string) ([]SSnapshotPolicyDisk, error) { - - return m.fetchAll(ctx, userCred, m.GetSlaveFieldName(), snapshotPolicyID) -} - -func (m *SSnapshotPolicyDiskManager) FetchDiskCountBySPID(snapshotpolicyID string) (int, error) { - disksq := DiskManager.Query("id").IsFalse("pending_deleted").SubQuery() - sdsq := m.Query().SubQuery() - q := sdsq.Query().Join(disksq, sqlchemy.Equals(disksq.Field("id"), - sdsq.Field("disk_id"))).Equals("snapshotpolicy_id", snapshotpolicyID) - return q.CountWithError() -} - -func (m *SSnapshotPolicyDiskManager) fetchAll(ctx context.Context, userCred mcclient.TokenCredential, - fieldName string, fieldValue string) ([]SSnapshotPolicyDisk, error) { - - q := m.Query() - q.Equals(fieldName, fieldValue) - ret := make([]SSnapshotPolicyDisk, 0) - err := db.FetchModelObjects(m, q, &ret) - if err != nil { - return nil, err - } - return ret, nil -} - -// ==================================================== sync =========================================================== - -// SyncDetachByDisk will detach all snapshot policies fo cloudRelations -// if cloudRelations is nil, it will detach all shapshot policies which has been attached to disk. -func (m *SSnapshotPolicyDiskManager) SyncDetachByDisk(ctx context.Context, userCred mcclient.TokenCredential, - cloudRelations []SSnapshotPolicyDisk, disk *SDisk) error { - snapshotPolicyDisks := cloudRelations - var err error - if snapshotPolicyDisks == nil { - snapshotPolicyDisks, err = m.FetchAllByDiskID(ctx, userCred, disk.GetId()) - if err != nil { - return errors.Wrapf(err, "Fetach allsnapshotpolicy of disk %s in database", disk.GetId()) - } - } - failResult := make([]string, 0, 1) - for i := range snapshotPolicyDisks { - err = snapshotPolicyDisks[i].DetachByDisk(ctx, userCred, disk) - if err != nil { - failResult = append(failResult, snapshotPolicyDisks[i].GetId()) - } - } - if len(failResult) != 0 { - errInfo := "detach failed which IDs are: " - return errors.Error(errInfo + strings.Join(failResult, ", ")) - } - return nil -} - -// SyncDetachBySnapshotpolicy detach all sn -func (m *SSnapshotPolicyDiskManager) SyncDetachBySnapshotpolicy(ctx context.Context, - userCred mcclient.TokenCredential, cloudRelations []SSnapshotPolicyDisk, snapshotPolicy *SSnapshotPolicy) error { - - snapshotPolicyDisks := cloudRelations - var err error - if snapshotPolicyDisks == nil { - snapshotPolicyDisks, err = m.FetchAllBySnapshotpolicyID(ctx, userCred, snapshotPolicy.GetId()) - if err != nil { - return errors.Wrapf(err, "Fetach all bysnapshotpolicy %s in database", snapshotPolicy.GetId()) - } - } - failResult := make([]string, 0, 1) - for i := range snapshotPolicyDisks { - err = snapshotPolicyDisks[i].DetachBySnapshotpolicy(ctx, userCred, snapshotPolicy) - if err != nil { - failResult = append(failResult, snapshotPolicyDisks[i].GetId()) - } - } - if len(failResult) != 0 { - errInfo := "detach failed which IDs are " - return errors.Error(errInfo + strings.Join(failResult, ", ")) - } - return nil -} - -func (m *SSnapshotPolicyDiskManager) SyncByDisk(ctx context.Context, userCred mcclient.TokenCredential, - extSnapshotpolicies []string, syncOwnerID mcclient.IIdentityProvider, disk *SDisk, storage *SStorage) error { - - sds, err := m.FetchAllByDiskID(ctx, userCred, disk.GetId()) - if err != nil { - return errors.Wrapf(err, "Fetach allsnapshotpolicy of disk %s in database", disk.GetId()) - } - - //fetch snapshotPolicy Cache to find the snapshotpolicyID corresponding to extSnapshotpolicyID - region, _ := storage.GetRegion() - spCaches, err := SnapshotPolicyCacheManager.FetchAllByExtIds(extSnapshotpolicies, region.GetId(), - storage.ManagerId) - if err != nil { - return errors.Wrapf(err, "fetachsnapshotpolicy caches failed") - } - cloudRelationsSet := make(map[string]struct{}) - for i := range spCaches { - cloudRelationsSet[spCaches[i].SnapshotpolicyId] = struct{}{} - } - - removed := make([]SSnapshotPolicyDisk, 0, 1) - added := make([]string, 0, 1) - for i := range sds { - if _, ok := cloudRelationsSet[sds[i].SnapshotpolicyId]; !ok { - removed = append(removed, sds[i]) - } - delete(cloudRelationsSet, sds[i].SnapshotpolicyId) - } - for k := range cloudRelationsSet { - added = append(added, k) - } - err = m.SyncDetachByDisk(ctx, userCred, removed, disk) - if err != nil { - return err - } - err = m.SyncAttachDisk(ctx, userCred, added, syncOwnerID, disk) - if err != nil { - return err - } - return nil -} - -func (m *SSnapshotPolicyDiskManager) SyncAttachDisk(ctx context.Context, userCred mcclient.TokenCredential, - Snapshotpolicies []string, syncOwnerID mcclient.IIdentityProvider, disk *SDisk) error { - - lockman.LockRawObject(ctx, "snapshot-policies", disk.Id) - defer lockman.ReleaseRawObject(ctx, "snapshot-policies", disk.Id) - - failRecord := make([]string, 0, 1) - for _, spId := range Snapshotpolicies { - snapshotpolicyDisk, err := db.FetchJointByIds(m, disk.GetId(), spId, jsonutils.JSONNull) - if err != nil && err != sql.ErrNoRows { - failRecord = append(failRecord, - fmt.Sprintf("Get SnapshotpolicyDisk whose diskid %s snapshotpolicyid %s failed", - disk.GetId(), spId)) - continue - } - if snapshotpolicyDisk != nil { - continue - } - - sd := SSnapshotPolicyDisk{} - sd.DiskId = disk.GetId() - sd.SnapshotpolicyId = spId - sd.Status = api.SNAPSHOT_POLICY_DISK_READY - err = m.TableSpec().Insert(ctx, &sd) - if err != nil { - failRecord = append(failRecord, fmt.Sprintf("attachsnapshotpolicy %s to disk %s failed", - spId, disk.GetId())) - continue - } - } - if len(failRecord) == 0 { - return nil - } - buf := bytes.NewBufferString("sync attach extSnapshotpolicies to disk ") - buf.WriteString(disk.GetId()) - buf.WriteString("failed because that ") - for i := range failRecord { - buf.WriteString(failRecord[i]) - buf.WriteString(", ") - } - buf.Truncate(buf.Len() - 2) - return errors.Error(buf.String()) -} - -func (m *SSnapshotPolicyDiskManager) SyncAttachDiskExt(ctx context.Context, userCred mcclient.TokenCredential, - extSnapshotpolicies []string, syncOwnerID mcclient.IIdentityProvider, disk *SDisk, storage *SStorage) error { - - //fetch snapshotPolicy Cache to find the snapshotpolicyID corresponding to extSnapshotpolicyID - region, _ := storage.GetRegion() - spCaches, err := SnapshotPolicyCacheManager.FetchAllByExtIds(extSnapshotpolicies, region.GetId(), - storage.ManagerId) - if err != nil { - return errors.Wrapf(err, "fetachsnapshotpolicy caches failed") - } - - snapshotPolicie := make([]string, 0, 1) - for i := range spCaches { - snapshotPolicie = append(snapshotPolicie, spCaches[i].SnapshotpolicyId) - } - - return m.SyncAttachDisk(ctx, userCred, snapshotPolicie, syncOwnerID, disk) -} - -// ==================================================== detach ========================================================= - -func (sd *SSnapshotPolicyDisk) RealDetach(ctx context.Context, userCred mcclient.TokenCredential) error { - return db.DeleteModel(ctx, userCred, sd) -} - -func (sd *SSnapshotPolicyDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { - return nil -} - -func (sd *SSnapshotPolicyDisk) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { - return nil -} - -// syncDetach should lock before -func (sd *SSnapshotPolicyDisk) DetachByDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *SDisk) error { - snapshotPolicy := SSnapshotPolicy{} - snapshotPolicy.Id = sd.SnapshotpolicyId - snapshotPolicy.SetModelManager(SnapshotPolicyManager, &snapshotPolicy) - lockman.LockJointObject(ctx, disk, &snapshotPolicy) - defer lockman.ReleaseJointObject(ctx, disk, &snapshotPolicy) - // todo call real Detach - return sd.RealDetach(ctx, userCred) -} - -func (sd *SSnapshotPolicyDisk) DetachBySnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential, - snapshotPolicy *SSnapshotPolicy) error { - - disk := SDisk{} - disk.Id = sd.DiskId - disk.SetModelManager(DiskManager, &disk) - lockman.LockJointObject(ctx, &disk, snapshotPolicy) - defer lockman.ReleaseJointObject(ctx, &disk, snapshotPolicy) - return sd.RealDetach(ctx, userCred) -} - -// ==================================================== create ========================================================= - -const ErrExistSD = errors.Error("snapshotpolicy disk has been exist") - -func (self *SSnapshotPolicyDiskManager) newSnapshotpolicyDisk(ctx context.Context, userCred mcclient.TokenCredential, - sp *SSnapshotPolicy, disk *SDisk) (*SSnapshotPolicyDisk, error) { - - q := self.Query().Equals("snapshotpolicy_id", sp.GetId()).Equals("disk_id", disk.GetId()) - count, err := q.CountWithError() - if err != nil { - return nil, nil - } - if count > 0 { - spd := SSnapshotPolicyDisk{} - q.First(&spd) - spd.SetModelManager(self, &spd) - return &spd, ErrExistSD - } - - spd := SSnapshotPolicyDisk{} - spd.SnapshotpolicyId = sp.GetId() - spd.DiskId = disk.GetId() - now := time.Now() - spd.NextSyncTime = sp.ComputeNextSyncTime(now) - spd.SetModelManager(self, &spd) - - lockman.LockJointObject(ctx, disk, sp) - defer lockman.ReleaseJointObject(ctx, disk, sp) - return &spd, self.TableSpec().Insert(ctx, &spd) - -} -func (self *SSnapshotPolicyDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - diskId, _ := data.GetString(self.GetMasterFieldName()) - snapshotPolicyId, _ := data.GetString(self.GetSlaveFieldName()) - disk := DiskManager.FetchDiskById(diskId) - snapshotPolicy, err := SnapshotPolicyManager.FetchSnapshotPolicyById(snapshotPolicyId) - if err != nil { - return nil, err - } - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - err = region.GetDriver().ValidateCreateSnapshopolicyDiskData(ctx, userCred, disk, snapshotPolicy) - if err != nil { - return nil, err - } - //to control that one disk should only bind one snapshot policy - spds, err := SnapshotPolicyDiskManager.FetchAllByDiskID(ctx, userCred, diskId) - if err != nil { - return nil, err - } - if len(spds) > 1 { - return nil, httperrors.NewInputParameterError("disk %s has too many snapshot policy attached", diskId) - } - if len(spds) == 1 { - data.Add(jsonutils.NewString(spds[0].SnapshotpolicyId), "need_detach") - } - // I don't want to request to database again behind - data.Add(jsonutils.Marshal(snapshotPolicy), "snapshotPolicy") - data.Add(jsonutils.Marshal(disk), "disk") - return data, nil -} - -func (sd *SSnapshotPolicyDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - sp, err := SnapshotPolicyManager.FetchSnapshotPolicyById(sd.SnapshotpolicyId) - if err != nil { - return err - } - now := time.Now() - sd.NextSyncTime = sp.ComputeNextSyncTime(now) - return nil -} - -func (sd *SSnapshotPolicyDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient. - IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { - - taskdata := data.(*jsonutils.JSONDict) - taskdata.Add(jsonutils.Marshal(sd), "snapshotPolicyDisk") - disk := &SDisk{} - data.Unmarshal(disk, "disk") - disk.SetModelManager(DiskManager, disk) - task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyApplyTask", disk, userCred, nil, "", "", nil) +func (self *SSnapshotPolicyDisk) GetDisk() (*SDisk, error) { + disk, err := DiskManager.FetchById(self.DiskId) if err != nil { - log.Errorf("SnapshotPolicyApplyTask newTask error %s", err) - } else { - task.ScheduleRun(taskdata) + return nil, errors.Wrapf(err, "FetchById(%s)", self.DiskId) } + return disk.(*SDisk), nil } -// ==================================================== delete ========================================================= - -func (sd *SSnapshotPolicyDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, - data jsonutils.JSONObject) error { - diskID := sd.DiskId - model, err := DiskManager.FetchById(diskID) - if err != nil { - return errors.Wrapf(err, "Fetch disk by ID %s failed", diskID) - } - disk := model.(*SDisk) - snapshotPolicyID := sd.SnapshotpolicyId - taskData := jsonutils.NewDict() - taskData.Add(jsonutils.NewString(snapshotPolicyID), "snapshot_policy_id") - taskData.Add(jsonutils.Marshal(sd), "snapshotPolicyDisk") - sd.SetStatus(userCred, api.SNAPSHOT_POLICY_DISK_DELETING, "") - task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCancelTask", disk, userCred, nil, "", "", nil) +func (self *SSnapshotPolicyDisk) GetSnapshotPolicy() (*SSnapshotPolicy, error) { + policy, err := SnapshotPolicyManager.FetchById(self.SnapshotpolicyId) if err != nil { - return errors.Wrapf(err, "SnapshotPolicyCancelTask newTask error %s", err) - } else { - task.ScheduleRun(taskData) + return nil, errors.Wrapf(err, "FetchById(%s)", self.SnapshotpolicyId) } - return nil + return policy.(*SSnapshotPolicy), nil } -func (manager *SSnapshotPolicyDiskManager) ListItemFilter( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyDiskListInput, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = manager.SVirtualJointResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualJointResourceBaseListInput) - if err != nil { - return nil, errors.Wrap(err, "SVirtualJointResourceBaseManager.ListItemFilter") - } - q, err = manager.SSnapshotPolicyResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SnapshotPolicyFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SSnapshotPolicyResourceBaseManager.ListItemFilter") - } - q, err = manager.SDiskResourceBaseManager.ListItemFilter(ctx, q, userCred, query.DiskFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SDiskResourceBaseManager.ListItemFilter") - } - - if len(query.Status) > 0 { - q = q.In("status", query.Status) - } - - return q, nil +func (man *SSnapshotPolicyDiskManager) RemoveByDisk(id string) error { + _, err := sqlchemy.GetDB().Exec( + fmt.Sprintf( + "delete from %s where disk_id = ?", + man.TableSpec().Name(), + ), id, + ) + return err } -func (manager *SSnapshotPolicyDiskManager) OrderByExtraFields( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyDiskListInput, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = manager.SVirtualJointResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.VirtualJointResourceBaseListInput) - if err != nil { - return nil, errors.Wrap(err, "SVirtualJointResourceBaseManager.OrderByExtraFields") - } - q, err = manager.SSnapshotPolicyResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SnapshotPolicyFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SSnapshotPolicyResourceBaseManager.OrderByExtraFields") - } - q, err = manager.SDiskResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.DiskFilterListInput) - if err != nil { - return nil, errors.Wrap(err, "SDiskResourceBaseManager.OrderByExtraFields") - } - - return q, nil -} - -func (manager *SSnapshotPolicyDiskManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { - var err error - - q, err = manager.SVirtualJointResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - q, err = manager.SSnapshotPolicyResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - q, err = manager.SDiskResourceBaseManager.QueryDistinctExtraField(q, field) - if err == nil { - return q, nil - } - - return q, httperrors.ErrNotFound -} - -func (manager *SSnapshotPolicyDiskManager) ListItemExportKeys(ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - keys stringutils2.SSortedStrings, -) (*sqlchemy.SQuery, error) { - var err error - - q, err = manager.SVirtualJointResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SVirtualJointResourceBaseManager.ListItemExportKeys") - } - if keys.ContainsAny(manager.SSnapshotPolicyResourceBaseManager.GetExportKeys()...) { - q, err = manager.SSnapshotPolicyResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SSnapshotPolicyResourceBaseManager.ListItemExportKeys") - } - } - if keys.ContainsAny(manager.SDiskResourceBaseManager.GetExportKeys()...) { - q, err = manager.SDiskResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) - if err != nil { - return nil, errors.Wrap(err, "SDiskResourceBaseManager.ListItemExportKeys") - } - } - - return q, nil +func (man *SSnapshotPolicyDiskManager) RemoveBySnapshotpolicy(id string) error { + _, err := sqlchemy.GetDB().Exec( + fmt.Sprintf( + "delete from %s where snapshotpolicy_id = ?", + man.TableSpec().Name(), + ), id, + ) + return err } diff --git a/pkg/compute/models/snapshotpolicyresource.go b/pkg/compute/models/snapshotpolicyresource.go deleted file mode 100644 index e76faab3080..00000000000 --- a/pkg/compute/models/snapshotpolicyresource.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "context" - "database/sql" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/reflectutils" - "yunion.io/x/sqlchemy" - - api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/stringutils2" -) - -type SSnapshotPolicyResourceBase struct { - // 本地快照策略ID - SnapshotpolicyId string `width:"36" charset:"ascii" nullable:"false" create:"required" index:"true" list:"user" json:"snapshotpolicy_id"` -} - -type SSnapshotPolicyResourceBaseManager struct{} - -func ValidateSnapshotPolicyResourceInput(userCred mcclient.TokenCredential, query api.SnapshotPolicyResourceInput) (*SSnapshotPolicy, api.SnapshotPolicyResourceInput, error) { - snapPolicyObj, err := SnapshotPolicyManager.FetchByIdOrName(userCred, query.SnapshotpolicyId) - if err != nil { - if err == sql.ErrNoRows { - return nil, query, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SnapshotPolicyManager.Keyword(), query.SnapshotpolicyId) - } else { - return nil, query, errors.Wrap(err, "SnapshotPolicyManager.FetchByIdOrName") - } - } - query.SnapshotpolicyId = snapPolicyObj.GetId() - return snapPolicyObj.(*SSnapshotPolicy), query, nil -} - -func (self *SSnapshotPolicyResourceBase) GetSnapshotPolicy() *SSnapshotPolicy { - spObj, err := SnapshotPolicyManager.FetchById(self.SnapshotpolicyId) - if err != nil { - log.Errorf("failed to find snapshot policy %s error: %v", self.SnapshotpolicyId, err) - return nil - } - return spObj.(*SSnapshotPolicy) -} - -func (manager *SSnapshotPolicyResourceBaseManager) FetchCustomizeColumns( - ctx context.Context, - userCred mcclient.TokenCredential, - query jsonutils.JSONObject, - objs []interface{}, - fields stringutils2.SSortedStrings, - isList bool, -) []api.SnapshotPolicyResourceInfo { - rows := make([]api.SnapshotPolicyResourceInfo, len(objs)) - snapshotPolicyIds := make([]string, len(objs)) - for i := range objs { - var base *SSnapshotPolicyResourceBase - err := reflectutils.FindAnonymouStructPointer(objs[i], &base) - if err != nil { - log.Errorf("Cannot find SSnapshotPolicyResourceBase in object %s", objs[i]) - continue - } - snapshotPolicyIds[i] = base.SnapshotpolicyId - } - snapshotPolicyNames, err := db.FetchIdNameMap2(SnapshotPolicyManager, snapshotPolicyIds) - if err != nil { - log.Errorf("FetchIdNameMap2 fail %s", err) - return rows - } - for i := range rows { - if name, ok := snapshotPolicyNames[snapshotPolicyIds[i]]; ok { - rows[i].Snapshotpolicy = name - } - } - return rows -} - -func (manager *SSnapshotPolicyResourceBaseManager) ListItemFilter( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyFilterListInput, -) (*sqlchemy.SQuery, error) { - if len(query.SnapshotpolicyId) > 0 { - snapPObj, _, err := ValidateSnapshotPolicyResourceInput(userCred, query.SnapshotPolicyResourceInput) - if err != nil { - return nil, errors.Wrap(err, "ValidateSnapshotPolicyResourceInput") - } - q = q.Equals("snapshotpolicy_id", snapPObj.GetId()) - } - return q, nil -} - -func (manager *SSnapshotPolicyResourceBaseManager) OrderByExtraFields( - ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyFilterListInput, -) (*sqlchemy.SQuery, error) { - if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { - return q, nil - } - orderQ := SnapshotPolicyManager.Query("id") - orderSubQ := orderQ.SubQuery() - orderQ, orders, fields := manager.GetOrderBySubQuery(orderQ, orderSubQ, orderQ.Field("id"), userCred, query, nil, nil) - q = q.LeftJoin(orderSubQ, sqlchemy.Equals(q.Field("snapshotpolicy_id"), orderSubQ.Field("id"))) - q = db.OrderByFields(q, orders, fields) - return q, nil -} - -func (manager *SSnapshotPolicyResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { - if field == "snapshotpolicy" { - snapPolicyQuery := SnapshotPolicyManager.Query("name", "id").Distinct().SubQuery() - q.AppendField(snapPolicyQuery.Field("name", field)) - q = q.Join(snapPolicyQuery, sqlchemy.Equals(q.Field("snapshotpolicy_id"), snapPolicyQuery.Field("id"))) - q.GroupBy(snapPolicyQuery.Field("name")) - return q, nil - } - return q, httperrors.ErrNotFound -} - -func (manager *SSnapshotPolicyResourceBaseManager) GetOrderBySubQuery( - q *sqlchemy.SQuery, - subq *sqlchemy.SSubQuery, - joinField sqlchemy.IQueryField, - userCred mcclient.TokenCredential, - query api.SnapshotPolicyFilterListInput, - orders []string, - fields []sqlchemy.IQueryField, -) (*sqlchemy.SQuery, []string, []sqlchemy.IQueryField) { - if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { - return q, orders, fields - } - snapQ := SnapshotPolicyManager.Query().SubQuery() - q = q.LeftJoin(snapQ, sqlchemy.Equals(joinField, snapQ.Field("id"))) - q = q.AppendField(snapQ.Field("name").Label("snapshotpolicy")) - orders = append(orders, query.OrderBySnapshotpolicy) - fields = append(fields, subq.Field("snapshotpolicy")) - return q, orders, fields -} - -func (manager *SSnapshotPolicyResourceBaseManager) GetOrderByFields(query api.SnapshotPolicyFilterListInput) []string { - return []string{query.OrderBySnapshotpolicy} -} - -func (manager *SSnapshotPolicyResourceBaseManager) ListItemExportKeys(ctx context.Context, - q *sqlchemy.SQuery, - userCred mcclient.TokenCredential, - keys stringutils2.SSortedStrings, -) (*sqlchemy.SQuery, error) { - if keys.ContainsAny(manager.GetExportKeys()...) { - subq := SnapshotPolicyManager.Query("id", "name").SubQuery() - q = q.LeftJoin(subq, sqlchemy.Equals(q.Field("snapshotpolicy_id"), subq.Field("id"))) - if keys.Contains("snapshotpolicy") { - q = q.AppendField(subq.Field("name", "snapshotpolicy")) - } - } - return q, nil -} - -func (manager *SSnapshotPolicyResourceBaseManager) GetExportKeys() []string { - return []string{"snapshotpolicy"} -} diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 09141f68915..dd2e050414c 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -537,10 +537,9 @@ func (manager *SSnapshotManager) OnCreateComplete(ctx context.Context, items []d func (self *SSnapshot) StartSnapshotCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { task, err := taskman.TaskManager.NewTask(ctx, "SnapshotCreateTask", self, userCred, params, parentTaskId, "", nil) if err != nil { - return err + return errors.Wrapf(err, "NewTask") } - task.ScheduleRun(nil) - return nil + return task.ScheduleRun(nil) } func (self *SSnapshot) GetGuest() (*SGuest, error) { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 4e2905bbe1b..648557a3092 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -106,9 +106,8 @@ type ComputeOptions struct { DefaultMaxManualSnapshotCount int `default:"2" help:"Per Disk max manual snapshot count, default 2"` //snapshot policy options - RetentionDaysLimit int `default:"49" help:"Days of snapshot retention, default 49 days"` - TimePointsLimit int `default:"1" help:"time point of every days, default 1 point"` - RepeatWeekdaysLimit int `default:"7" help:"day point of every weekday, default 7 points"` + RetentionDaysLimit int `default:"49" help:"Days of snapshot retention, default 49 days"` + TimePointsLimit int `default:"1" help:"time point of every days, default 1 point"` ServerStatusSyncIntervalMinutes int `default:"5" help:"Interval to sync server status, defualt is 5 minutes"` diff --git a/pkg/compute/regiondrivers/aliyun.go b/pkg/compute/regiondrivers/aliyun.go index b485747a46f..4a906a87b12 100644 --- a/pkg/compute/regiondrivers/aliyun.go +++ b/pkg/compute/regiondrivers/aliyun.go @@ -194,24 +194,6 @@ func (self *SAliyunRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx cont return input, nil } -func (self *SAliyunRegionDriver) ValidateCreateSnapshopolicyDiskData(ctx context.Context, - userCred mcclient.TokenCredential, disk *models.SDisk, snapshotPolicy *models.SSnapshotPolicy) error { - //err := self.SManagedVirtualizationRegionDriver.ValidateCreateSnapshopolicyDiskData(ctx, userCred, disk, snapshotPolicy) - //if err != nil { - // return nil - //} - //// In Aliyun, One disk only apply one snapshot policy - //ret, err := models.SnapshotPolicyDiskManager.FetchAllByDiskID(ctx, userCred, disk.GetId()) - //if err != nil { - // return err - //} - //if len(ret) != 0 { - // return httperrors.NewBadRequestError("One disk could't attach two snapshot policy in aliyun; please detach last one first.") - //} - //return nil - return nil -} - func (self *SAliyunRegionDriver) ValidateCreateSnapshotData(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, storage *models.SStorage, input *api.SnapshotCreateInput) error { if strings.HasPrefix(input.Name, "auto") || strings.HasPrefix(input.Name, "http://") || strings.HasPrefix(input.Name, "https://") { return httperrors.NewBadRequestError( diff --git a/pkg/compute/regiondrivers/base.go b/pkg/compute/regiondrivers/base.go index 2e609762310..28e03a8024d 100644 --- a/pkg/compute/regiondrivers/base.go +++ b/pkg/compute/regiondrivers/base.go @@ -155,18 +155,6 @@ func (self *SBaseRegionDriver) RequestDeleteLoadbalancerListenerRule(ctx context return fmt.Errorf("Not Implement RequestDeleteLoadbalancerListenerRule") } -func (self *SBaseRegionDriver) RequestUpdateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, input cloudprovider.SnapshotPolicyInput, task taskman.ITask) error { - return fmt.Errorf("Not Implement RequestUpdateSnapshotPolicy") -} - -func (self *SBaseRegionDriver) RequestApplySnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - return fmt.Errorf("Not Implement RequestApplySnapshotPolicy") -} - -func (self *SBaseRegionDriver) RequestCancelSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - return fmt.Errorf("Not Implement RequestCancelSnapshotPolicy") -} - func (self *SBaseRegionDriver) ValidateSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot) error { return fmt.Errorf("Not Implement ValidateSnapshotDelete") } @@ -195,18 +183,6 @@ func (self *SBaseRegionDriver) OnDiskReset(ctx context.Context, userCred mcclien return fmt.Errorf("Not Implement OnDiskReset") } -func (self *SBaseRegionDriver) ValidateCreateSnapshopolicyDiskData(ctx context.Context, - userCred mcclient.TokenCredential, disk *models.SDisk, snapshotPolicy *models.SSnapshotPolicy) error { - - if disk.DomainId != snapshotPolicy.DomainId { - return httperrors.NewBadRequestError("disk and snapshotpolicy should have same domain") - } - if disk.ProjectId != snapshotPolicy.ProjectId { - return httperrors.NewBadRequestError("disk and snapshotpolicy should have same project") - } - return nil -} - func (self *SBaseRegionDriver) OnSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot, task taskman.ITask, data jsonutils.JSONObject) error { return fmt.Errorf("Not implement OnSnapshotDelete") } @@ -509,3 +485,23 @@ func (drv *SBaseRegionDriver) GetSecurityGroupFilter(vpc *models.SVpc) (func(q * func (self *SBaseRegionDriver) ValidateUpdateSecurityGroupRuleInput(ctx context.Context, userCred mcclient.TokenCredential, input *api.SSecgroupRuleUpdateInput) (*api.SSecgroupRuleUpdateInput, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "ValidateUpdateSecurityGroupInput") } + +func (self *SBaseRegionDriver) ValidateCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, input *api.SSnapshotPolicyCreateInput) (*api.SSnapshotPolicyCreateInput, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "ValidateCreateSnapshotPolicy") +} + +func (self *SBaseRegionDriver) RequestCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestCreateSnapshotPolicy") +} + +func (self *SBaseRegionDriver) RequestDeleteSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestDeleteSnapshotPolicy") +} + +func (self *SBaseRegionDriver) RequestSnapshotPolicyBindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestSnapshotPolicyBindDisks") +} + +func (self *SBaseRegionDriver) RequestSnapshotPolicyUnbindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestSnapshotPolicyUnbindDisks") +} diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index dc030ad07e5..d37bfaa59b3 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -40,7 +41,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -846,57 +846,6 @@ func (self *SKVMRegionDriver) OnDiskReset(ctx context.Context, userCred mcclient return models.GetStorageDriver(storage.StorageType).OnDiskReset(ctx, userCred, disk, snapshot, data) } -func (self *SKVMRegionDriver) RequestUpdateSnapshotPolicy(ctx context.Context, - userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, input cloudprovider.SnapshotPolicyInput, - task taskman.ITask) error { - - return nil -} - -func (self *SKVMRegionDriver) ValidateCreateSnapshopolicyDiskData(ctx context.Context, - userCred mcclient.TokenCredential, disk *models.SDisk, snapshotPolicy *models.SSnapshotPolicy) error { - - err := self.SBaseRegionDriver.ValidateCreateSnapshopolicyDiskData(ctx, userCred, disk, snapshotPolicy) - if err != nil { - return err - } - - if snapshotPolicy.RetentionDays < -1 || snapshotPolicy.RetentionDays == 0 || snapshotPolicy.RetentionDays > options.Options.RetentionDaysLimit { - return httperrors.NewInputParameterError("Retention days must in 1~%d or -1", options.Options.RetentionDaysLimit) - } - - repeatWeekdays := models.SnapshotPolicyManager.RepeatWeekdaysToIntArray(snapshotPolicy.RepeatWeekdays) - timePoints := models.SnapshotPolicyManager.TimePointsToIntArray(snapshotPolicy.TimePoints) - - if len(repeatWeekdays) > options.Options.RepeatWeekdaysLimit { - return httperrors.NewInputParameterError("repeat_weekdays only contains %d days at most", - options.Options.RepeatWeekdaysLimit) - } - - if len(timePoints) > options.Options.TimePointsLimit { - return httperrors.NewInputParameterError("time_points only contains %d points at most", options.Options.TimePointsLimit) - } - return nil -} - -func (self *SKVMRegionDriver) RequestApplySnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(sp.GetId()), "snapshotpolicy_id") - return data, nil - }) - return nil -} - -func (self *SKVMRegionDriver) RequestCancelSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(sp.GetId()), "snapshotpolicy_id") - return data, nil - }) - return nil -} - func (self *SKVMRegionDriver) OnSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot, task taskman.ITask, data jsonutils.JSONObject) error { task.SetStage("OnKvmSnapshotDelete", nil) task.ScheduleRun(data) @@ -1197,16 +1146,6 @@ func (self *SKVMRegionDriver) RequestAssociateEipForNAT(ctx context.Context, use return errors.Wrapf(cloudprovider.ErrNotSupported, "RequestAssociateEipForNAT") } -func (self *SKVMRegionDriver) RequestPreSnapshotPolicyApply(ctx context.Context, userCred mcclient. - TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - return data, nil - }) - return nil -} - func (self *SKVMRegionDriver) ValidateCacheSecgroup(ctx context.Context, userCred mcclient.TokenCredential, secgroup *models.SSecurityGroup, vpc *models.SVpc, classic bool) error { return errors.Wrap(httperrors.ErrNotSupported, "No need to cache secgroup for onecloud region") } @@ -1642,3 +1581,38 @@ func (self *SKVMRegionDriver) ValidateUpdateSecurityGroupRuleInput(ctx context.C return input, nil } + +func (self *SKVMRegionDriver) ValidateCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, input *api.SSnapshotPolicyCreateInput) (*api.SSnapshotPolicyCreateInput, error) { + return input, nil +} + +func (self *SKVMRegionDriver) RequestCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + sp.SetStatus(userCred, apis.STATUS_AVAILABLE, "") + return task.ScheduleRun(nil) +} + +func (self *SKVMRegionDriver) RequestDeleteSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + return task.ScheduleRun(nil) +} + +func (self *SKVMRegionDriver) RequestSnapshotPolicyBindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + disks, err := sp.GetUnbindDisks(diskIds) + if err != nil { + return nil, errors.Wrapf(err, "GetUnbindDisks") + } + ids := []string{} + for _, disk := range disks { + ids = append(ids, disk.Id) + } + return nil, sp.BindDisks(ctx, disks) + }) + return nil +} + +func (self *SKVMRegionDriver) RequestSnapshotPolicyUnbindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + return nil, sp.UnbindDisks(diskIds) + }) + return nil +} diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index a2bd97345e2..eec30ba2f1d 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -33,6 +33,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -1038,90 +1039,6 @@ func (self *SManagedVirtualizationRegionDriver) RequestDeleteVpc(ctx context.Con return nil } -func (self *SManagedVirtualizationRegionDriver) RequestUpdateSnapshotPolicy(ctx context.Context, userCred mcclient. - TokenCredential, sp *models.SSnapshotPolicy, input cloudprovider.SnapshotPolicyInput, task taskman.ITask) error { - // it's too cumbersome to pass parameters in taskman, so change a simple way for the moment - - //spcache, err := models.SnapshotPolicyCacheManager.FetchSnapshotPolicyCache(sp.GetId(), sp.CloudregionId, sp.ManagerId) - //if err != nil { - // return errors.Wrapf(err, "Fetch cache ofsnapshotpolicy %s", sp.GetId()) - //} - //return spcache.UpdateCloudSnapshotPolicy(&input) - - return nil -} - -// RequestApplySnapshotPolicy apply snapshotpolicy for public cloud. -// In our system, one disk only can hava one snapshot policy attached. -// Default, some public cloud such as Aliyun is same with us and this function shoule be used for these public cloud. -// But in Some public cloud such as Qcloud different with us, -// we should wirte a new function in corressponding regiondriver which detach all snapshotpolicy of disk after -// attache new one. -// You can refer to the implementations of function SQcloudRegionDriver.RequestApplySnapshotPolicy(). -func (self *SManagedVirtualizationRegionDriver) RequestApplySnapshotPolicy(ctx context.Context, - userCred mcclient.TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, - data jsonutils.JSONObject) error { - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - regionId := region.GetId() - providerId := storage.ManagerId - spcache, err := models.SnapshotPolicyCacheManager.Register(ctx, userCred, sp.GetId(), regionId, providerId) - if err != nil { - return nil, errors.Wrap(err, "registersnapshotpolicy cache failed") - } - - iRegion, err := disk.GetIRegion(ctx) - if err != nil { - return nil, err - } - - err = iRegion.ApplySnapshotPolicyToDisks(spcache.GetExternalId(), disk.GetExternalId()) - if err != nil { - return nil, err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(sp.GetId()), "snapshotpolicy_id") - return data, nil - }) - return nil -} - -func (self *SManagedVirtualizationRegionDriver) RequestCancelSnapshotPolicy(ctx context.Context, userCred mcclient. - TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - regionId := region.GetId() - providerId := storage.ManagerId - spcache, err := models.SnapshotPolicyCacheManager.FetchSnapshotPolicyCache(sp.GetId(), regionId, providerId) - - if err != nil { - return nil, errors.Wrap(err, "registersnapshotpolicy cache failed") - } - - iRegion, err := spcache.GetIRegion(ctx) - if err != nil { - return nil, err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(sp.GetId()), "snapshotpolicy_id") - err = iRegion.CancelSnapshotPolicyToDisks(spcache.GetExternalId(), disk.GetExternalId()) - if errors.Cause(err) == cloudprovider.ErrNotFound { - return data, nil - } - if err != nil { - return nil, err - } - return data, nil - }) - return nil -} - func (self *SManagedVirtualizationRegionDriver) ValidateSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot) error { return nil } @@ -1210,20 +1127,6 @@ func (self *SManagedVirtualizationRegionDriver) OnDiskReset(ctx context.Context, return nil } -func (self *SManagedVirtualizationRegionDriver) ValidateCreateSnapshopolicyDiskData(ctx context.Context, - userCred mcclient.TokenCredential, disk *models.SDisk, snapshotPolicy *models.SSnapshotPolicy) error { - - err := self.SBaseRegionDriver.ValidateCreateSnapshopolicyDiskData(ctx, userCred, disk, snapshotPolicy) - if err != nil { - return err - } - - if snapshotPolicy.RetentionDays < -1 || snapshotPolicy.RetentionDays == 0 || snapshotPolicy.RetentionDays > 65535 { - return httperrors.NewInputParameterError("Retention days must in 1~65535 or -1") - } - return nil -} - func (self *SManagedVirtualizationRegionDriver) OnSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot, task taskman.ITask, data jsonutils.JSONObject) error { task.SetStage("OnManagedSnapshotDelete", nil) task.ScheduleRun(data) @@ -1238,16 +1141,6 @@ func (self *SManagedVirtualizationRegionDriver) RequestAssociateEipForNAT(ctx co return eip.StartEipAssociateTask(ctx, userCred, jsonutils.Marshal(opts).(*jsonutils.JSONDict), task.GetTaskId()) } -func (self *SManagedVirtualizationRegionDriver) RequestPreSnapshotPolicyApply(ctx context.Context, userCred mcclient. - TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - return data, nil - }) - return nil -} - func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstance(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *models.SDBInstance, task taskman.ITask) error { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { iregion, err := dbinstance.GetIRegion(ctx) @@ -3543,3 +3436,124 @@ func (self *SManagedVirtualizationRegionDriver) CreateDefaultSecurityGroup( } return newGroup, nil } + +func (self *SManagedVirtualizationRegionDriver) ValidateCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, input *api.SSnapshotPolicyCreateInput) (*api.SSnapshotPolicyCreateInput, error) { + if len(input.CloudproviderId) == 0 { + return nil, httperrors.NewMissingParameterError("cloudprovider_id") + } + managerObj, err := validators.ValidateModel(userCred, models.CloudproviderManager, &input.CloudproviderId) + if err != nil { + return nil, err + } + input.ManagerId = input.CloudproviderId + manager := managerObj.(*models.SCloudprovider) + if manager.Provider != region.Provider { + return nil, httperrors.NewConflictError("manager %s is not %s cloud", manager.Name, region.Provider) + } + return input, nil +} + +func (self *SManagedVirtualizationRegionDriver) RequestCreateSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iRegion, err := sp.GetIRegion(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetIRegion") + } + opts := &cloudprovider.SnapshotPolicyInput{ + Name: sp.Name, + Desc: sp.Description, + RetentionDays: sp.RetentionDays, + TimePoints: sp.TimePoints, + RepeatWeekdays: sp.RepeatWeekdays, + } + opts.Tags, _ = sp.GetAllUserMetadata() + id, err := iRegion.CreateSnapshotPolicy(opts) + if err != nil { + return nil, errors.Wrapf(err, "CreateSnapshotPolicy") + } + _, err = db.Update(sp, func() error { + sp.ExternalId = id + sp.Status = apis.STATUS_AVAILABLE + return nil + }) + return nil, err + }) + return nil +} + +func (self *SManagedVirtualizationRegionDriver) RequestDeleteSnapshotPolicy(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, sp *models.SSnapshotPolicy, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iPolicy, err := sp.GetISnapshotPolicy(ctx) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows || errors.Cause(err) == cloudprovider.ErrNotFound { + return nil, nil + } + return nil, errors.Wrapf(err, "GetISnapshotPolicy") + } + + err = iPolicy.Delete() + if err != nil { + return nil, errors.Wrapf(err, "Delete") + } + + return nil, nil + }) + return nil + +} + +func (self *SManagedVirtualizationRegionDriver) RequestSnapshotPolicyBindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iPolicy, err := sp.GetISnapshotPolicy(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetISnapshotPolicy") + } + disks, err := sp.GetUnbindDisks(diskIds) + if err != nil { + return nil, errors.Wrapf(err, "GetUnbindDisks") + } + externalIds := []string{} + for _, disk := range disks { + if len(disk.ExternalId) > 0 && !utils.IsInStringArray(disk.ExternalId, externalIds) { + externalIds = append(externalIds, disk.ExternalId) + } + } + if len(externalIds) > 0 { + err = iPolicy.ApplyDisks(externalIds) + if err != nil { + return nil, errors.Wrapf(err, "ApplyDisks %s", externalIds) + } + return nil, sp.BindDisks(ctx, disks) + } + return nil, nil + }) + return nil +} + +func (self *SManagedVirtualizationRegionDriver) RequestSnapshotPolicyUnbindDisks(ctx context.Context, userCred mcclient.TokenCredential, sp *models.SSnapshotPolicy, diskIds []string, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iPolicy, err := sp.GetISnapshotPolicy(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetISnapshotPolicy") + } + disks, err := sp.GetBindDisks(diskIds) + if err != nil { + return nil, errors.Wrapf(err, "GetBindDisks") + } + externalIds := []string{} + for _, disk := range disks { + if len(disk.ExternalId) > 0 && !utils.IsInStringArray(disk.ExternalId, externalIds) { + externalIds = append(externalIds, disk.ExternalId) + } + } + if len(externalIds) > 0 { + err = iPolicy.CancelDisks(externalIds) + if err != nil { + return nil, errors.Wrapf(err, "CancelDisks %s", externalIds) + } + return nil, sp.UnbindDisks(diskIds) + } + return nil, nil + }) + return nil +} diff --git a/pkg/compute/regiondrivers/qcloud.go b/pkg/compute/regiondrivers/qcloud.go index ccdbbbe1949..0428d268e51 100644 --- a/pkg/compute/regiondrivers/qcloud.go +++ b/pkg/compute/regiondrivers/qcloud.go @@ -274,34 +274,6 @@ func (self *SQcloudRegionDriver) RequestSyncLoadbalancerBackend(ctx context.Cont return nil } -func (self *SQcloudRegionDriver) RequestPreSnapshotPolicyApply(ctx context.Context, userCred mcclient. - TokenCredential, task taskman.ITask, disk *models.SDisk, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) error { - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - if sp == nil { - return data, nil - } - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - spcache, err := models.SnapshotPolicyCacheManager.FetchSnapshotPolicyCache(sp.GetId(), - region.GetId(), storage.ManagerId) - if err != nil { - return nil, err - } - iRegion, err := spcache.GetIRegion(ctx) - if err != nil { - return nil, err - } - err = iRegion.CancelSnapshotPolicyToDisks(spcache.GetExternalId(), disk.GetExternalId()) - if err != nil { - return nil, err - } - return data, nil - }) - return nil -} - func (self *SQcloudRegionDriver) InitDBInstanceUser(ctx context.Context, instance *models.SDBInstance, task taskman.ITask, desc *cloudprovider.SManagedDBInstanceCreateConfig) error { user := "root" account := models.SDBInstanceAccount{} diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 3435f33d8c3..c6f0bd97a98 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -98,6 +98,8 @@ func InitHandlers(app *appsrv.Application) { models.WafRuleStatementManager, models.BillingResourceCheckManager, + + models.SnapshotPolicyDiskManager, } { db.RegisterModelManager(manager) } @@ -139,7 +141,6 @@ func InitHandlers(app *appsrv.Application) { models.InstanceSnapshotManager, models.SnapshotManager, models.SnapshotPolicyManager, - models.SnapshotPolicyCacheManager, models.BaremetalagentManager, models.LoadbalancerManager, models.LoadbalancerListenerManager, @@ -271,7 +272,6 @@ func InitHandlers(app *appsrv.Application) { models.CloudproviderRegionManager, models.DBInstanceNetworkManager, models.NetworkinterfacenetworkManager, - models.SnapshotPolicyDiskManager, models.InstanceSnapshotJointManager, models.DnsZoneVpcManager, models.DBInstanceSecgroupManager, diff --git a/pkg/compute/tasks/disk_clean_overdued_snapshots.go b/pkg/compute/tasks/disk_clean_overdued_snapshots.go index 3728ad7da4c..ad920ad8a1a 100644 --- a/pkg/compute/tasks/disk_clean_overdued_snapshots.go +++ b/pkg/compute/tasks/disk_clean_overdued_snapshots.go @@ -40,12 +40,7 @@ func init() { func (self *DiskCleanOverduedSnapshots) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { disk := obj.(*models.SDisk) - spId, _ := self.Params.GetString("snapshotpolicy_id") - sp, _ := models.SnapshotPolicyManager.FetchSnapshotPolicyById(spId) - if sp == nil { - self.SetStageFailed(ctx, jsonutils.NewString("missing snapshot policy ???")) - return - } + retentionDays, _ := self.Params.Int("retention_days") now, err := self.Params.GetTime("start_time") if err != nil { @@ -66,8 +61,8 @@ func (self *DiskCleanOverduedSnapshots) OnInit(ctx context.Context, obj db.IStan } cleanOverdueSnapshots = snapCount > (options.Options.DefaultMaxSnapshotCount - options.Options.DefaultMaxManualSnapshotCount) - if sp.RetentionDays > 0 && !cleanOverdueSnapshots { - t := now.AddDate(0, 0, -1*sp.RetentionDays) + if retentionDays > 0 && !cleanOverdueSnapshots { + t := now.AddDate(0, 0, -1*int(retentionDays)) snapCount, err = models.SnapshotManager.Query().Equals("fake_deleted", false).Equals("disk_id", disk.Id). Equals("created_by", compute.SNAPSHOT_AUTO).LT("created_at", t).CountWithError() if err != nil { diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 39bb69e0ad0..1e50ce087b6 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -177,12 +177,13 @@ func (self *DiskDeleteTask) startPendingDeleteDisk(ctx context.Context, disk *mo self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("pending delete disk failed")) return } - err = models.SnapshotPolicyDiskManager.SyncDetachByDisk(ctx, self.UserCred, nil, disk) + err = models.SnapshotPolicyDiskManager.RemoveByDisk(disk.Id) if err != nil { self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("detach all snapshotpolicies of disk failed")) return } + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/snapshot_policy_cache_delete_task.go b/pkg/compute/tasks/snapshot_policy_cache_delete_task.go deleted file mode 100644 index c949e75db7f..00000000000 --- a/pkg/compute/tasks/snapshot_policy_cache_delete_task.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tasks - -import ( - "context" - - "yunion.io/x/jsonutils" - - api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/compute/models" -) - -type SnapshotPolicyCacheDeleteTask struct { - taskman.STask -} - -func init() { - taskman.RegisterTask(SnapshotPolicyCacheDeleteTask{}) -} - -func (self *SnapshotPolicyCacheDeleteTask) taskFailed(ctx context.Context, cache *models.SSnapshotPolicyCache, - err jsonutils.JSONObject) { - - cache.SetStatus(self.UserCred, api.SNAPSHOT_POLICY_CACHE_STATUS_DELETE_FAILED, err.String()) - self.SetStageFailed(ctx, err) -} - -func (self *SnapshotPolicyCacheDeleteTask) taskComplete(ctx context.Context, cache *models.SSnapshotPolicyCache) { - cache.RealDetele(ctx, self.UserCred) - self.SetStageComplete(ctx, nil) -} - -func (self *SnapshotPolicyCacheDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, - data jsonutils.JSONObject) { - - cache := obj.(*models.SSnapshotPolicyCache) - - if len(cache.ExternalId) == 0 { - self.taskComplete(ctx, cache) - return - } - - err := cache.DeleteCloudSnapshotPolicy(ctx) - if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) - return - } - self.taskComplete(ctx, cache) -} diff --git a/pkg/compute/tasks/snapshotpolicy_bind_disks_task.go b/pkg/compute/tasks/snapshotpolicy_bind_disks_task.go new file mode 100644 index 00000000000..3e162d0b69c --- /dev/null +++ b/pkg/compute/tasks/snapshotpolicy_bind_disks_task.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type SnapshotpolicyBindDisksTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(SnapshotpolicyBindDisksTask{}) +} + +func (self *SnapshotpolicyBindDisksTask) taskFailed(ctx context.Context, sp *models.SSnapshotPolicy, err error) { + sp.SetStatus(self.UserCred, api.SNAPSHOT_POLICY_APPLY_FAILED, err.Error()) + db.OpsLog.LogEvent(sp, db.ACT_BIND, err, self.UserCred) + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_BIND, err, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (self *SnapshotpolicyBindDisksTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + sp := obj.(*models.SSnapshotPolicy) + + region, err := sp.GetRegion() + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "GetRegion")) + return + } + + diskIds := jsonutils.GetQueryStringArray(self.Params, "disk_ids") + + self.SetStage("OnSnapshotPolicyBindDisksComplete", nil) + err = region.GetDriver().RequestSnapshotPolicyBindDisks(ctx, self.UserCred, sp, diskIds, self) + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "RequestSnapshotPolicyBindDisks")) + return + } +} + +func (self *SnapshotpolicyBindDisksTask) OnSnapshotPolicyBindDisksComplete(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + sp.SetStatus(self.UserCred, apis.STATUS_AVAILABLE, "") + self.SetStageComplete(ctx, nil) +} + +func (self *SnapshotpolicyBindDisksTask) OnSnapshotPolicyBindDisksCompleteFailed(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + self.taskFailed(ctx, sp, errors.Errorf(data.String())) +} diff --git a/pkg/compute/tasks/snapshotpolicy_create_task.go b/pkg/compute/tasks/snapshotpolicy_create_task.go new file mode 100644 index 00000000000..c0a4490f8b9 --- /dev/null +++ b/pkg/compute/tasks/snapshotpolicy_create_task.go @@ -0,0 +1,73 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type SnapshotPolicyCreateTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(SnapshotPolicyCreateTask{}) +} + +func (self *SnapshotPolicyCreateTask) taskFailed(ctx context.Context, sp *models.SSnapshotPolicy, err error) { + sp.SetStatus(self.UserCred, apis.STATUS_CREATE_FAILED, err.Error()) + db.OpsLog.LogEvent(sp, db.ACT_ALLOCATE_FAIL, err, self.UserCred) + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_ALLOCATE, err, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (self *SnapshotPolicyCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + sp := obj.(*models.SSnapshotPolicy) + + region, err := sp.GetRegion() + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "GetRegion")) + return + } + self.SetStage("OnCreateSnapshotPolicyComplete", nil) + err = region.GetDriver().RequestCreateSnapshotPolicy(ctx, self.UserCred, region, sp, self) + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "RequestCreateSnapshotPolicy")) + return + } +} + +func (self *SnapshotPolicyCreateTask) OnCreateSnapshotPolicyComplete(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_ALLOCATE, nil, self.UserCred, true) + notifyclient.EventNotify(ctx, self.UserCred, notifyclient.SEventNotifyParam{ + Obj: sp, + Action: notifyclient.ActionCreate, + }) + self.SetStageComplete(ctx, nil) +} + +func (self *SnapshotPolicyCreateTask) OnCreateSnapshotPolicyCompleteFailed(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + self.taskFailed(ctx, sp, errors.Errorf(data.String())) +} diff --git a/pkg/compute/tasks/snapshot_policy_delete_task.go b/pkg/compute/tasks/snapshotpolicy_delete_task.go similarity index 70% rename from pkg/compute/tasks/snapshot_policy_delete_task.go rename to pkg/compute/tasks/snapshotpolicy_delete_task.go index 6af8d992b35..2d3d86aa38a 100644 --- a/pkg/compute/tasks/snapshot_policy_delete_task.go +++ b/pkg/compute/tasks/snapshotpolicy_delete_task.go @@ -18,8 +18,9 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" - api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -35,22 +36,29 @@ func init() { taskman.RegisterTask(SnapshotPolicyDeleteTask{}) } -func (self *SnapshotPolicyDeleteTask) taskFail(ctx context.Context, sp *models.SSnapshotPolicy, reason jsonutils.JSONObject) { - sp.SetStatus(self.GetUserCred(), api.SNAPSHOT_POLICY_DELETE_FAILED, reason.String()) - db.OpsLog.LogEvent(sp, db.ACT_DELOCATE_FAIL, reason, self.UserCred) - logclient.AddActionLogWithStartable(self, sp, logclient.ACT_DELOCATE, reason, self.UserCred, false) - notifyclient.NotifySystemErrorWithCtx(ctx, sp.Id, sp.Name, api.SNAPSHOT_POLICY_DELETE_FAILED, reason.String()) - self.SetStageFailed(ctx, reason) +func (self *SnapshotPolicyDeleteTask) taskFail(ctx context.Context, sp *models.SSnapshotPolicy, err error) { + sp.SetStatus(self.GetUserCred(), apis.STATUS_DELETE_FAILED, err.Error()) + db.OpsLog.LogEvent(sp, db.ACT_DELOCATE_FAIL, err, self.UserCred) + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_DELOCATE, err, self.UserCred, false) + notifyclient.NotifySystemErrorWithCtx(ctx, sp.Id, sp.Name, apis.STATUS_DELETE_FAILED, err.Error()) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) } func (self *SnapshotPolicyDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { sp := obj.(*models.SSnapshotPolicy) - err := models.SnapshotPolicyCacheManager.DeleteCloudSnapshotPolices(ctx, self.UserCred, sp.GetId()) + + region, err := sp.GetRegion() + if err != nil { + self.taskFail(ctx, sp, errors.Wrapf(err, "GetRegion")) + return + } + + self.SetStage("OnSnapshotPolicyDeleteComplete", nil) + err = region.GetDriver().RequestDeleteSnapshotPolicy(ctx, self.UserCred, region, sp, self) if err != nil { - self.taskFail(ctx, sp, jsonutils.NewString(err.Error())) + self.taskFail(ctx, sp, errors.Wrapf(err, "RequestDeleteSnapshotPolicy")) return } - self.OnSnapshotPolicyDeleteComplete(ctx, sp, data) } func (self *SnapshotPolicyDeleteTask) OnSnapshotPolicyDeleteComplete(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { @@ -65,5 +73,5 @@ func (self *SnapshotPolicyDeleteTask) OnSnapshotPolicyDeleteComplete(ctx context } func (self *SnapshotPolicyDeleteTask) OnSnapshotPolicyDeleteCompleteFailed(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { - self.taskFail(ctx, sp, data) + self.taskFail(ctx, sp, errors.Errorf(data.String())) } diff --git a/pkg/compute/tasks/snapshotpolicy_disk_task.go b/pkg/compute/tasks/snapshotpolicy_disk_task.go deleted file mode 100644 index cf72c7ffe0e..00000000000 --- a/pkg/compute/tasks/snapshotpolicy_disk_task.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tasks - -import ( - "context" - "fmt" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - - "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/logclient" -) - -func init() { - taskman.RegisterTask(SnapshotPolicyApplyTask{}) - taskman.RegisterTask(SnapshotPolicyCancelTask{}) -} - -type SnapshotPolicyApplyTask struct { - taskman.STask -} - -func (self *SnapshotPolicyApplyTask) taskFail(ctx context.Context, disk *models.SDisk, - spd *models.SSnapshotPolicyDisk, reason jsonutils.JSONObject) { - - if spd != nil { - err := spd.RealDetach(ctx, self.UserCred) - if err != nil { - log.Errorf("Delete snapshotpolicydisk %s failed, need to delete", spd.GetId()) - } - snapshotpolicy, err := models.SnapshotPolicyManager.FetchSnapshotPolicyById(spd.SnapshotpolicyId) - if err == nil { - db.OpsLog.LogEvent(snapshotpolicy, db.ACT_SNAPSHOT_POLICY_BIND_DISK_FAIL, "", self.UserCred) - logclient.AddActionLogWithStartable(self, snapshotpolicy, logclient.ACT_BIND_DISK, "", self.UserCred, false) - } - } - - disk.SetStatus(self.UserCred, compute.DISK_APPLY_SNAPSHOT_FAIL, reason.String()) - - db.OpsLog.LogEvent(disk, db.ACT_APPLY_SNAPSHOT_POLICY_FAILED, reason, self.UserCred) - logclient.AddActionLogWithStartable(self, disk, logclient.ACT_APPLY_SNAPSHOT_POLICY, reason, self.UserCred, false) - notifyclient.NotifySystemErrorWithCtx(ctx, disk.GetId(), disk.Name, compute.DISK_APPLY_SNAPSHOT_FAIL, reason.String()) - self.SetStageFailed(ctx, reason) -} - -func (self *SnapshotPolicyApplyTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - disk := obj.(*models.SDisk) - spd := models.SSnapshotPolicyDisk{} - data.Unmarshal(&spd, "snapshotPolicyDisk") - - var snapshotPolicy *models.SSnapshotPolicy - if data.Contains("need_detach") { - snapshotPolicyID, _ := data.GetString("need_detach") - model, err := models.SnapshotPolicyManager.FetchById(snapshotPolicyID) - if err != nil { - self.taskFail(ctx, disk, &spd, jsonutils.NewString(err.Error())) - return - } - snapshotPolicy = model.(*models.SSnapshotPolicy) - } - self.Params.Add(jsonutils.NewString(spd.SnapshotpolicyId), "snapshotpolicy_id") - - self.SetStage("OnPreSnapshotPolicyApplyComplete", nil) - // pass data to next Stage without inserting database through this way - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - if err := region.GetDriver().RequestPreSnapshotPolicyApply(ctx, self.UserCred, self, disk, snapshotPolicy, - data); err != nil { - - self.taskFail(ctx, disk, &spd, jsonutils.NewString(err.Error())) - return - } -} - -func (self *SnapshotPolicyApplyTask) OnPreSnapshotPolicyApplyCompleteFailed(ctx context.Context, disk *models.SDisk, - reason jsonutils.JSONObject) { - - spId, _ := self.Params.GetString("snapshotpolicy_id") - spd, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(spId, disk.GetId()) - if err != nil { - self.taskFail(ctx, disk, nil, reason) - return - } - self.taskFail(ctx, disk, spd, reason) -} - -func (self *SnapshotPolicyApplyTask) OnPreSnapshotPolicyApplyComplete(ctx context.Context, disk *models.SDisk, - data jsonutils.JSONObject) { - - snapshotPolicy := models.SSnapshotPolicy{} - spd := models.SSnapshotPolicyDisk{} - data.Unmarshal(&snapshotPolicy, "snapshotPolicy") - data.Unmarshal(&spd, "snapshotPolicyDisk") - - if data.Contains("need_detach") { - snapshotPolicyID, _ := data.GetString("need_detach") - spd1, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(snapshotPolicyID, disk.GetId()) - if err != nil { - self.taskFail(ctx, disk, &spd, jsonutils.NewString(err.Error())) - return - } - if spd1 != nil { - spd1.RealDetach(ctx, self.UserCred) - } - } - self.SetStage("OnSnapshotPolicyApply", nil) - - // pass data to next Stage without inserting database through this way - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - if err := region.GetDriver(). - RequestApplySnapshotPolicy(ctx, self.UserCred, self, disk, &snapshotPolicy, data); err != nil { - - self.taskFail(ctx, disk, &spd, jsonutils.NewString(err.Error())) - } -} - -func (self *SnapshotPolicyApplyTask) OnSnapshotPolicyApplyFailed(ctx context.Context, disk *models.SDisk, - reason jsonutils.JSONObject) { - - spId, _ := self.Params.GetString("snapshotpolicy_id") - spd, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(spId, disk.GetId()) - if err != nil { - self.taskFail(ctx, disk, nil, reason) - return - } - self.taskFail(ctx, disk, spd, reason) -} - -func (self *SnapshotPolicyApplyTask) OnSnapshotPolicyApply(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { - sp_id, _ := data.GetString("snapshotpolicy_id") - spd, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(sp_id, disk.GetId()) - if err != nil { - log.Errorf("Fetch snapshotpolicy disk failed") - } else if spd != nil { - spd.SetStatus(self.UserCred, compute.SNAPSHOT_POLICY_DISK_READY, "") - snapshotpolicy, err := models.SnapshotPolicyManager.FetchSnapshotPolicyById(spd.SnapshotpolicyId) - if err == nil { - db.OpsLog.LogEvent(snapshotpolicy, db.ACT_SNAPSHOT_POLICY_BIND_DISK, "", self.UserCred) - logclient.AddActionLogWithStartable(self, snapshotpolicy, logclient.ACT_BIND_DISK, "", self.UserCred, true) - } - } - db.OpsLog.LogEvent(disk, db.ACT_APPLY_SNAPSHOT_POLICY, "", self.UserCred) - logclient.AddActionLogWithStartable(self, disk, logclient.ACT_APPLY_SNAPSHOT_POLICY, "", self.UserCred, true) - self.SetStageComplete(ctx, nil) -} - -// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - -type SnapshotPolicyCancelTask struct { - taskman.STask -} - -func (self *SnapshotPolicyCancelTask) taskFail(ctx context.Context, disk *models.SDisk, spd *models.SSnapshotPolicyDisk, reason jsonutils.JSONObject) { - if spd != nil { - spd.SetStatus(self.UserCred, compute.SNAPSHOT_POLICY_DISK_DELETE_FAILED, "") - snapshotpolicy, err := models.SnapshotPolicyManager.FetchSnapshotPolicyById(spd.SnapshotpolicyId) - if err == nil { - db.OpsLog.LogEvent(snapshotpolicy, db.ACT_SNAPSHOT_POLICY_UNBIND_DISK_FAIL, "", self.UserCred) - logclient.AddActionLogWithStartable(self, snapshotpolicy, logclient.ACT_UNBIND_DISK, "", self.UserCred, false) - } - } - disk.SetStatus(self.UserCred, compute.DISK_CALCEL_SNAPSHOT_FAIL, reason.String()) - - db.OpsLog.LogEvent(disk, db.ACT_CANCEL_SNAPSHOT_POLICY_FAILED, reason, self.UserCred) - logclient.AddActionLogWithStartable(self, disk, logclient.ACT_CANCEL_SNAPSHOT_POLICY, reason, self.UserCred, false) - self.SetStageFailed(ctx, reason) -} - -func (self *SnapshotPolicyCancelTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - disk := obj.(*models.SDisk) - snapshotPolicyID, _ := data.GetString("snapshot_policy_id") - spd := models.SSnapshotPolicyDisk{} - data.Unmarshal(&spd, "snapshotPolicyDisk") - self.Params.Add(jsonutils.NewString(snapshotPolicyID), "snapshotpolicy_id") - - model, err := models.SnapshotPolicyManager.FetchById(snapshotPolicyID) - if err != nil { - self.taskFail(ctx, disk, &spd, jsonutils.NewString(fmt.Sprintf("failed to fetch disk by id %s: %s", snapshotPolicyID, err.Error()))) - return - } - snapshotPolicy := model.(*models.SSnapshotPolicy) - self.SetStage("OnSnapshotPolicyCancel", nil) - storage, _ := disk.GetStorage() - region, _ := storage.GetRegion() - if err := region.GetDriver().RequestCancelSnapshotPolicy(ctx, self.UserCred, self, disk, snapshotPolicy, data); err != nil { - - self.taskFail(ctx, disk, &spd, jsonutils.NewString(err.Error())) - // fmt.Sprintf("faile to detach snapshot policy %s and disk %s: %s", snapshotPolicy.Id, disk.Id, err.Error())) - } -} - -func (self *SnapshotPolicyCancelTask) OnSnapshotPolicyCancelFailed(ctx context.Context, disk *models.SDisk, - reason jsonutils.JSONObject) { - - spId, _ := self.Params.GetString("snapshotpolicy_id") - spd, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(spId, disk.GetId()) - if err != nil { - self.taskFail(ctx, disk, nil, reason) - return - } - self.taskFail(ctx, disk, spd, reason) -} - -func (self *SnapshotPolicyCancelTask) OnSnapshotPolicyCancel(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { - sp_id, _ := data.GetString("snapshotpolicy_id") - spd, err := models.SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(sp_id, disk.GetId()) - if err != nil { - log.Errorf("Fechsnapshotpolicy disk failed") - } - if spd != nil { - //real detach - spd.RealDetach(ctx, self.UserCred) - snapshotpolicy, err := models.SnapshotPolicyManager.FetchSnapshotPolicyById(spd.SnapshotpolicyId) - if err == nil { - db.OpsLog.LogEvent(snapshotpolicy, db.ACT_SNAPSHOT_POLICY_UNBIND_DISK, "", self.UserCred) - logclient.AddActionLogWithStartable(self, snapshotpolicy, logclient.ACT_UNBIND_DISK, "", self.UserCred, true) - } - } - - db.OpsLog.LogEvent(disk, db.ACT_CANCEL_SNAPSHOT_POLICY, "", self.UserCred) - logclient.AddActionLogWithStartable(self, disk, logclient.ACT_CANCEL_SNAPSHOT_POLICY, "", self.UserCred, true) - self.SetStageComplete(ctx, nil) -} diff --git a/pkg/compute/tasks/snapshotpolicy_syncstatus_task.go b/pkg/compute/tasks/snapshotpolicy_syncstatus_task.go new file mode 100644 index 00000000000..e7b4e903003 --- /dev/null +++ b/pkg/compute/tasks/snapshotpolicy_syncstatus_task.go @@ -0,0 +1,62 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type SnapshotpolicySyncstatusTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(SnapshotpolicySyncstatusTask{}) +} + +func (self *SnapshotpolicySyncstatusTask) taskFailed(ctx context.Context, sp *models.SSnapshotPolicy, err error) { + sp.SetStatus(self.UserCred, apis.STATUS_UNKNOWN, err.Error()) + db.OpsLog.LogEvent(sp, db.ACT_SYNC_STATUS, err, self.UserCred) + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_SYNC_STATUS, err, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (self *SnapshotpolicySyncstatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + sp := obj.(*models.SSnapshotPolicy) + + iSp, err := sp.GetISnapshotPolicy(ctx) + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "GetISnapshotPolicy")) + return + } + + provider, err := sp.GetCloudprovider() + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "GetCloudprovider")) + return + } + + sp.SyncWithCloudPolicy(ctx, self.UserCred, provider, iSp) + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/snapshotpolicy_unbind_disks_task.go b/pkg/compute/tasks/snapshotpolicy_unbind_disks_task.go new file mode 100644 index 00000000000..bf0b041565d --- /dev/null +++ b/pkg/compute/tasks/snapshotpolicy_unbind_disks_task.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type SnapshotpolicyUnbindDisksTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(SnapshotpolicyUnbindDisksTask{}) +} + +func (self *SnapshotpolicyUnbindDisksTask) taskFailed(ctx context.Context, sp *models.SSnapshotPolicy, err error) { + sp.SetStatus(self.UserCred, api.SNAPSHOT_POLICY_CANCEL_FAILED, err.Error()) + db.OpsLog.LogEvent(sp, db.ACT_UNBIND, err, self.UserCred) + logclient.AddActionLogWithStartable(self, sp, logclient.ACT_UNBIND, err, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) +} + +func (self *SnapshotpolicyUnbindDisksTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + sp := obj.(*models.SSnapshotPolicy) + + region, err := sp.GetRegion() + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "GetRegion")) + return + } + + diskIds := jsonutils.GetQueryStringArray(self.Params, "disk_ids") + + self.SetStage("OnSnapshotPolicyUnbindDisksComplete", nil) + err = region.GetDriver().RequestSnapshotPolicyUnbindDisks(ctx, self.UserCred, sp, diskIds, self) + if err != nil { + self.taskFailed(ctx, sp, errors.Wrapf(err, "RequestSnapshotPolicyUnbindDisks")) + return + } +} + +func (self *SnapshotpolicyUnbindDisksTask) OnSnapshotPolicyUnbindDisksComplete(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + sp.SetStatus(self.UserCred, apis.STATUS_AVAILABLE, "") + self.SetStageComplete(ctx, nil) +} + +func (self *SnapshotpolicyUnbindDisksTask) OnSnapshotPolicyUnbindDisksCompleteFailed(ctx context.Context, sp *models.SSnapshotPolicy, data jsonutils.JSONObject) { + self.taskFailed(ctx, sp, errors.Errorf(data.String())) +} diff --git a/pkg/mcclient/modules/compute/mod_snapshotpolicycache.go b/pkg/mcclient/modules/compute/mod_snapshotpolicycache.go deleted file mode 100644 index 0d3bc3439ba..00000000000 --- a/pkg/mcclient/modules/compute/mod_snapshotpolicycache.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import ( - "yunion.io/x/onecloud/pkg/mcclient/modulebase" - "yunion.io/x/onecloud/pkg/mcclient/modules" -) - -var SnapshotPolicyCache modulebase.ResourceManager - -func init() { - SnapshotPolicyCache = modules.NewComputeManager("snapshotpolicycache", "snapshotpolicycaches", - []string{"Snapshotpolicy_Id", "External_Id", "Cloudregion_Id", "Manager_Id"}, - []string{}, - ) - - modules.RegisterCompute(&SnapshotPolicyCache) -} diff --git a/pkg/mcclient/modules/compute/mod_snapshotpolicydisk.go b/pkg/mcclient/modules/compute/mod_snapshotpolicydisk.go deleted file mode 100644 index 5bcc152f4f9..00000000000 --- a/pkg/mcclient/modules/compute/mod_snapshotpolicydisk.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import ( - "yunion.io/x/onecloud/pkg/mcclient/modulebase" - "yunion.io/x/onecloud/pkg/mcclient/modules" -) - -var ( - SnapshotPolicyDisk modulebase.JointResourceManager - SnapshotPolicyDisk1 modulebase.JointResourceManager -) - -func init() { - SnapshotPolicyDisk = modules.NewJointComputeManager( - "snapshotpolicydisk", - "snapshotpolicydisks", - []string{"Disk_ID", "Snapshotpolicy_ID"}, - []string{}, - &Disks, - &SnapshotPoliciy) - modules.RegisterCompute(&SnapshotPolicyDisk) - - SnapshotPolicyDisk1 = modules.NewJointComputeManager( - "snapshotpolicydisk", - "snapshotpolicydisks", - []string{"Disk_ID", "Snapshotpolicy_ID"}, - []string{}, - &SnapshotPoliciy, - &Disks) - modules.RegisterCompute(&SnapshotPolicyDisk1) -} diff --git a/pkg/mcclient/options/compute/snapshotpolicy.go b/pkg/mcclient/options/compute/snapshotpolicy.go new file mode 100644 index 00000000000..b88cee428ba --- /dev/null +++ b/pkg/mcclient/options/compute/snapshotpolicy.go @@ -0,0 +1,54 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type SnapshotPolicyListOptions struct { + options.BaseListOptions + + OrderByBindDiskCount string +} + +func (opts *SnapshotPolicyListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(opts) +} + +type SnapshotPolicyCreateOptions struct { + options.BaseCreateOptions + CloudregionId string + ManagerId string + + RetentionDays int `help:"snapshot retention days" default:"-1"` + RepeatWeekdays []int `help:"snapshot create days on week"` + TimePoints []int `help:"snapshot create time points on one day"` +} + +func (opts *SnapshotPolicyCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + +type SnapshotPolicyDisksOptions struct { + options.BaseIdOptions + Disks []string `help:"ids of disk"` +} + +func (opts *SnapshotPolicyDisksOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(map[string]interface{}{"disks": opts.Disks}), nil +} diff --git a/pkg/util/logclient/consts.go b/pkg/util/logclient/consts.go index 4c02617e173..3f607ff2576 100644 --- a/pkg/util/logclient/consts.go +++ b/pkg/util/logclient/consts.go @@ -261,6 +261,7 @@ const ( ACT_DISSOCIATE = "dissociate" ACT_BIND = "bind" + ACT_UNBIND = "unbind" ACT_PROGRESS = "progress" ACT_ADD_BASTION_SERVER = "add_bastion_server" diff --git a/vendor/modules.txt b/vendor/modules.txt index b573d186602..42eff38b684 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1465,7 +1465,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231214030208-02cfe93ac519 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231215084755-601e4d02e207 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/snapshot_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/snapshot_const.go index 6b5a0917567..2399c970155 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/snapshot_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/snapshot_const.go @@ -23,10 +23,5 @@ const ( SNAPSHOT_DELETING = "deleting" SNAPSHOT_UNKNOWN = "unknown" - SNAPSHOT_POLICY_CREATING = "creating" - - SNAPSHOT_POLICY_READY = "ready" - SNAPSHOT_POLICY_UNKNOWN = "unknown" - INSTANCE_SNAPSHOT_READY = "ready" ) diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/const.go index 3d9595404dd..206242cb350 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/const.go @@ -20,6 +20,7 @@ const ( STATUS_CREATING = "creating" STATUS_CREATE_FAILED = "create_failed" STATUS_AVAILABLE = "available" + STATUS_UNKNOWN = "unknown" USER_TAG_PREFIX = "user:" diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go index 202a05e5b26..30a5f513da8 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go @@ -32,6 +32,7 @@ const ( CLOUD_CAPABILITY_NETWORK = "network" CLOUD_CAPABILITY_SECURITY_GROUP = "security_group" CLOUD_CAPABILITY_EIP = "eip" + CLOUD_CAPABILITY_SNAPSHOT_POLICY = "snapshot_policy" CLOUD_CAPABILITY_LOADBALANCER = "loadbalancer" CLOUD_CAPABILITY_OBJECTSTORE = "objectstore" CLOUD_CAPABILITY_RDS = "rds" diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go index 26c6c167b55..8762b93365d 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go @@ -99,12 +99,8 @@ type ICloudRegion interface { GetISnapshotById(snapshotId string) (ICloudSnapshot, error) CreateSnapshotPolicy(*SnapshotPolicyInput) (string, error) - UpdateSnapshotPolicy(*SnapshotPolicyInput, string) error - DeleteSnapshotPolicy(string) error - ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error - CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error GetISnapshotPolicies() ([]ICloudSnapshotPolicy, error) - GetISnapshotPolicyById(snapshotPolicyId string) (ICloudSnapshotPolicy, error) + GetISnapshotPolicyById(id string) (ICloudSnapshotPolicy, error) GetIHosts() ([]ICloudHost, error) GetIHostById(id string) (ICloudHost, error) @@ -545,8 +541,6 @@ type ICloudDisk interface { CreateISnapshot(ctx context.Context, name string, desc string) (ICloudSnapshot, error) GetISnapshots() ([]ICloudSnapshot, error) - GetExtSnapshotPolicyIds() ([]string, error) - Resize(ctx context.Context, newSizeMB int64) error Reset(ctx context.Context, snapshotId string) (string, error) @@ -574,10 +568,13 @@ type ICloudInstanceSnapshot interface { type ICloudSnapshotPolicy interface { IVirtualResource - IsActivated() bool GetRetentionDays() int GetRepeatWeekdays() ([]int, error) GetTimePoints() ([]int, error) + Delete() error + ApplyDisks(ids []string) error + CancelDisks(ids []string) error + GetApplyDiskIds() ([]string, error) } type ICloudGlobalVpc interface { diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/snapshot_policy.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/snapshot_policy.go index 434c5f6d88f..528d5fed763 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/snapshot_policy.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/snapshot_policy.go @@ -17,9 +17,12 @@ package cloudprovider import "strconv" type SnapshotPolicyInput struct { - RetentionDays int - RepeatWeekdays, TimePoints []int - PolicyName string + RetentionDays int + RepeatWeekdays []int + TimePoints []int + Name string + Desc string + Tags map[string]string } func (spi *SnapshotPolicyInput) GetStringArrayRepeatWeekdays() []string { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go index 7cadcaefdbb..53f213d71de 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go @@ -1032,6 +1032,7 @@ func (region *SAliyunClient) GetCapabilities() []string { cloudprovider.CLOUD_CAPABILITY_CONTAINER + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_TABLESTORE + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_CERT, + cloudprovider.CLOUD_CAPABILITY_SNAPSHOT_POLICY, } return caps } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/disk.go index c6329f145cb..d08c7e95543 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/disk.go @@ -75,14 +75,10 @@ func (self *SDisk) GetIops() int { return self.IOPS } -func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) { - if limit > 50 || limit <= 0 { - limit = 50 - } +func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, snapshotpolicyId string) ([]SDisk, error) { params := make(map[string]string) params["RegionId"] = self.RegionId - params["PageSize"] = fmt.Sprintf("%d", limit) - params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + params["MaxResults"] = "500" if len(instanceId) > 0 { params["InstanceId"] = instanceId @@ -93,24 +89,38 @@ func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, if len(category) > 0 { params["Category"] = category } - if diskIds != nil && len(diskIds) > 0 { + if len(diskIds) > 0 { params["DiskIds"] = jsonutils.Marshal(diskIds).String() } - - body, err := self.ecsRequest("DescribeDisks", params) - if err != nil { - log.Errorf("GetDisks fail %s", err) - return nil, 0, err + if len(snapshotpolicyId) > 0 { + params["AutoSnapshotPolicyId"] = snapshotpolicyId } - disks := make([]SDisk, 0) - err = body.Unmarshal(&disks, "Disks", "Disk") - if err != nil { - log.Errorf("Unmarshal disk details fail %s", err) - return nil, 0, err + ret := []SDisk{} + for { + resp, err := self.ecsRequest("DescribeDisks", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeDisks") + } + + part := struct { + Disks struct { + Disk []SDisk + } + NextToken string + }{} + + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + ret = append(ret, part.Disks.Disk...) + if len(part.NextToken) == 0 || len(part.Disks.Disk) == 0 { + break + } + params["NextToken"] = part.NextToken } - total, _ := body.Int("TotalCount") - return disks, int(total), nil + return ret, nil } func (self *SDisk) GetId() string { @@ -278,14 +288,16 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz } func (self *SRegion) getDisk(diskId string) (*SDisk, error) { - disks, total, err := self.GetDisks("", "", "", []string{diskId}, 0, 1) + disks, err := self.GetDisks("", "", "", []string{diskId}, "") if err != nil { return nil, err } - if total != 1 { - return nil, cloudprovider.ErrNotFound + for i := range disks { + if disks[i].DiskId == diskId { + return &disks[i], nil + } } - return &disks[0], nil + return nil, errors.Wrapf(cloudprovider.ErrNotFound, diskId) } func (self *SRegion) DeleteDisk(diskId string) error { @@ -391,13 +403,6 @@ func (self *SDisk) GetCreatedAt() time.Time { return self.CreationTime } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - if len(self.AutoSnapshotPolicyId) == 0 { - return []string{}, nil - } - return []string{self.AutoSnapshotPolicyId}, nil -} - func (self *SDisk) GetExpiredAt() time.Time { return convertExpiredAt(self.ExpiredTime) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/instance.go index 34f1a6d2c44..5642a3dbf9b 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/instance.go @@ -243,16 +243,9 @@ func (a byAttachedTime) Less(i, j int) bool { } func (self *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { - disks := []SDisk{} - for { - part, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, len(disks), 50) - if err != nil { - return nil, errors.Wrapf(err, "GetDisks for %s", self.InstanceId) - } - disks = append(disks, part...) - if len(disks) >= total { - break - } + disks, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, "") + if err != nil { + return nil, errors.Wrapf(err, "GetDisks for %s", self.InstanceId) } sort.Sort(byAttachedTime(disks)) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/snapshot_policy.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/snapshot_policy.go index 393e6b36d92..e36c76be4f6 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/snapshot_policy.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/snapshot_policy.go @@ -20,8 +20,9 @@ import ( "strconv" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" - api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/apis" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud" ) @@ -56,25 +57,27 @@ func (self *SSnapshotPolicy) GetName() string { } func (self *SSnapshotPolicy) GetStatus() string { - // XXX: aliyun文档与实际返回值不符 - if self.Status == Normal || self.Status == Available { - return api.SNAPSHOT_POLICY_READY - } else if self.Status == Creating { - return api.SNAPSHOT_POLICY_CREATING - } else { - return api.SNAPSHOT_POLICY_UNKNOWN + switch self.Status { + case Normal, Available: + return apis.STATUS_AVAILABLE + case Creating: + return apis.STATUS_CREATING + default: + return apis.STATUS_UNKNOWN } } func (self *SSnapshotPolicy) Refresh() error { - if snapshotPolicies, total, err := self.region.GetSnapshotPolicies(self.AutoSnapshotPolicyId, 0, 1); err != nil { - return err - } else if total != 1 { - return cloudprovider.ErrNotFound - } else if err := jsonutils.Update(self, snapshotPolicies[0]); err != nil { - return err + policies, err := self.region.GetSnapshotPolicies(self.AutoSnapshotPolicyId) + if err != nil { + return errors.Wrapf(err, "GetSnapshotPolicies") } - return nil + for i := range policies { + if policies[i].AutoSnapshotPolicyId == self.AutoSnapshotPolicyId { + return jsonutils.Update(self, policies[i]) + } + } + return errors.Wrapf(cloudprovider.ErrNotFound, self.AutoSnapshotPolicyId) } func (self *SSnapshotPolicy) IsEmulated() bool { @@ -134,57 +137,57 @@ func (self *SSnapshotPolicy) GetTimePoints() ([]int, error) { return parsePolicy(self.TimePoints) } -func (self *SSnapshotPolicy) IsActivated() bool { - return true -} - func (self *SRegion) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) { - snapshotPolicies, total, err := self.GetSnapshotPolicies("", 0, 50) + policies, err := self.GetSnapshotPolicies("") if err != nil { return nil, err } - for len(snapshotPolicies) < total { - var parts []SSnapshotPolicy - parts, total, err = self.GetSnapshotPolicies("", len(snapshotPolicies), 50) - if err != nil { - return nil, err - } - snapshotPolicies = append(snapshotPolicies, parts...) - } - ret := make([]cloudprovider.ICloudSnapshotPolicy, len(snapshotPolicies)) - for i := 0; i < len(snapshotPolicies); i += 1 { - ret[i] = &snapshotPolicies[i] + ret := make([]cloudprovider.ICloudSnapshotPolicy, len(policies)) + for i := 0; i < len(policies); i += 1 { + ret[i] = &policies[i] } return ret, nil } -func (self *SRegion) GetSnapshotPolicies(policyId string, offset int, limit int) ([]SSnapshotPolicy, int, error) { +func (self *SRegion) GetSnapshotPolicies(policyId string) ([]SSnapshotPolicy, error) { params := make(map[string]string) params["RegionId"] = self.RegionId - if limit != 0 { - params["PageSize"] = fmt.Sprintf("%d", limit) - params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) - } + params["PageSize"] = "100" + pageNum := 1 + params["PageNumber"] = fmt.Sprintf("%d", pageNum) if len(policyId) > 0 { params["AutoSnapshotPolicyId"] = policyId } - body, err := self.ecsRequest("DescribeAutoSnapshotPolicyEx", params) - if err != nil { - return nil, 0, fmt.Errorf("GetSnapshotPolicys fail %s", err) - } - - snapshotPolicies := make([]SSnapshotPolicy, 0) - if err := body.Unmarshal(&snapshotPolicies, "AutoSnapshotPolicies", "AutoSnapshotPolicy"); err != nil { - return nil, 0, fmt.Errorf("Unmarshal snapshot policies details fail %s", err) + ret := make([]SSnapshotPolicy, 0) + for { + resp, err := self.ecsRequest("DescribeAutoSnapshotPolicyEx", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeAutoSnapshotPolicyEx") + } + part := struct { + TotalCount int + AutoSnapshotPolicies struct { + AutoSnapshotPolicy []SSnapshotPolicy + } + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, err + } + ret = append(ret, part.AutoSnapshotPolicies.AutoSnapshotPolicy...) + if len(ret) >= part.TotalCount || len(part.AutoSnapshotPolicies.AutoSnapshotPolicy) == 0 { + break + } + pageNum++ + params["PageNumber"] = fmt.Sprintf("%d", pageNum) } - total, _ := body.Int("TotalCount") - for i := 0; i < len(snapshotPolicies); i += 1 { - snapshotPolicies[i].region = self + for i := 0; i < len(ret); i += 1 { + ret[i].region = self } - return snapshotPolicies, int(total), nil + return ret, nil } func (self *SSnapshotPolicy) Delete() error { @@ -202,15 +205,17 @@ func (self *SRegion) DeleteSnapshotPolicy(snapshotPolicyId string) error { return err } -func (self *SRegion) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) { - policies, _, err := self.GetSnapshotPolicies(snapshotPolicyId, 0, 1) +func (self *SRegion) GetISnapshotPolicyById(id string) (cloudprovider.ICloudSnapshotPolicy, error) { + policies, err := self.GetSnapshotPolicies(id) if err != nil { return nil, err } - if len(policies) == 0 { - return nil, cloudprovider.ErrNotFound + for i := range policies { + if policies[i].AutoSnapshotPolicyId == id { + return &policies[i], nil + } } - return &policies[0], nil + return nil, errors.Wrapf(cloudprovider.ErrNotFound, id) } func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput) (string, error) { @@ -225,12 +230,12 @@ func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp params["repeatWeekdays"] = jsonutils.Marshal(input.GetStringArrayRepeatWeekdays()).String() params["timePoints"] = jsonutils.Marshal(input.GetStringArrayTimePoints()).String() params["retentionDays"] = strconv.Itoa(input.RetentionDays) - params["autoSnapshotPolicyName"] = input.PolicyName - if body, err := self.ecsRequest("CreateAutoSnapshotPolicy", params); err != nil { - return "", fmt.Errorf("CreateAutoSnapshotPolicy fail %s", err) - } else { - return body.GetString("AutoSnapshotPolicyId") + params["autoSnapshotPolicyName"] = input.Name + body, err := self.ecsRequest("CreateAutoSnapshotPolicy", params) + if err != nil { + return "", errors.Wrapf(err, "CreateAutoSnapshotPolicy") } + return body.GetString("AutoSnapshotPolicyId") } func (self *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput, snapshotPolicyId string) error { @@ -252,52 +257,45 @@ func (self *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp return nil } -//func (self *SRegion) UpdateSnapshotPolicy( -// snapshotPolicyId string, retentionDays *int, -// repeatWeekdays, timePoints *jsonutils.JSONArray, policyName string, -//) error { -// params := make(map[string]string) -// params["RegionId"] = self.RegionId -// if len(policyName) > 0 { -// params["autoSnapshotPolicyName"] = policyName -// } -// if retentionDays != nil { -// params["retentionDays"] = strconv.Itoa(*retentionDays) -// } -// if repeatWeekdays != nil { -// params["repeatWeekdays"] = repeatWeekdays.String() -// } -// if timePoints != nil { -// params["timePoints"] = timePoints.String() -// } -// _, err := self.ecsRequest("ModifyAutoSnapshotPolicyEx", params) -// if err != nil { -// return fmt.Errorf("ModifyAutoSnapshotPolicyEx Fail %s", err) -// } -// return nil -//} - -func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["RegionId"] = self.RegionId params["autoSnapshotPolicyId"] = snapshotPolicyId - diskIds := []string{diskId} params["diskIds"] = jsonutils.Marshal(diskIds).String() _, err := self.ecsRequest("ApplyAutoSnapshotPolicy", params) if err != nil { - return fmt.Errorf("ApplyAutoSnapshotPolicy Fail %s", err) + return errors.Wrapf(err, "ApplyAutoSnapshotPolicy") } return nil } -func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SSnapshotPolicy) ApplyDisks(ids []string) error { + return self.region.ApplySnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} + +func (self *SSnapshotPolicy) GetApplyDiskIds() ([]string, error) { + disks, err := self.region.GetDisks("", "", "", nil, self.AutoSnapshotPolicyId) + if err != nil { + return nil, err + } + ret := []string{} + for _, disk := range disks { + ret = append(ret, disk.DiskId) + } + return ret, nil +} + +func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["RegionId"] = self.RegionId - diskIds := []string{diskId} params["diskIds"] = jsonutils.Marshal(diskIds).String() _, err := self.ecsRequest("CancelAutoSnapshotPolicy", params) if err != nil { - return fmt.Errorf("CancelAutoSnapshotPolicy Fail %s", err) + return errors.Wrapf(err, "CancelAutoSnapshotPolicy") } return nil } + +func (self *SSnapshotPolicy) CancelDisks(ids []string) error { + return self.region.CancelSnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/storage.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/storage.go index e299c50d1fe..42b7ad3f4b5 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/storage.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/storage.go @@ -69,39 +69,31 @@ func (self *SStorage) GetIZone() cloudprovider.ICloudZone { func (self *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) { disks := make([]SDisk, 0) - offset := 0 storageType := self.storageType if self.storageType == api.STORAGE_CLOUD_ESSD_PL2 || self.storageType == api.STORAGE_CLOUD_ESSD_PL3 || self.storageType == api.STORAGE_CLOUD_ESSD_PL0 { storageType = api.STORAGE_CLOUD_ESSD } - for { - parts, total, err := self.zone.region.GetDisks("", self.zone.GetId(), storageType, nil, offset, 50) - if err != nil { - return nil, errors.Wrapf(err, "GetDisks") - } - performanceLevel := "" - switch self.storageType { - case api.STORAGE_CLOUD_ESSD_PL0: - performanceLevel = "PL0" - case api.STORAGE_CLOUD_ESSD: - performanceLevel = "PL1" - case api.STORAGE_CLOUD_ESSD_PL2: - performanceLevel = "PL2" - case api.STORAGE_CLOUD_ESSD_PL3: - performanceLevel = "PL3" - } - for _, disk := range parts { - if disk.PerformanceLevel == performanceLevel { - disks = append(disks, disk) - } - } - - offset += len(parts) - - if offset >= total { - break + parts, err := self.zone.region.GetDisks("", self.zone.GetId(), storageType, nil, "") + if err != nil { + return nil, errors.Wrapf(err, "GetDisks") + } + performanceLevel := "" + switch self.storageType { + case api.STORAGE_CLOUD_ESSD_PL0: + performanceLevel = "PL0" + case api.STORAGE_CLOUD_ESSD: + performanceLevel = "PL1" + case api.STORAGE_CLOUD_ESSD_PL2: + performanceLevel = "PL2" + case api.STORAGE_CLOUD_ESSD_PL3: + performanceLevel = "PL3" + } + for _, disk := range parts { + if disk.PerformanceLevel == performanceLevel { + disks = append(disks, disk) } } + idisks := make([]cloudprovider.ICloudDisk, len(disks)) for i := 0; i < len(disks); i += 1 { disks[i].storage = self diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go index fe682f01742..e7fc95d354d 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go @@ -533,6 +533,7 @@ func (region *SApsaraClient) GetCapabilities() []string { cloudprovider.CLOUD_CAPABILITY_QUOTA + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_IPV6_GATEWAY + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_TABLESTORE + cloudprovider.READ_ONLY_SUFFIX, + cloudprovider.CLOUD_CAPABILITY_SNAPSHOT_POLICY, } return caps } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/disk.go index fc3236e09ef..9a191a5310e 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/disk.go @@ -76,14 +76,12 @@ type SDisk struct { DepartmentInfo } -func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) { - if limit > 50 || limit <= 0 { - limit = 50 - } +func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, snapshotpolicyId string) ([]SDisk, error) { params := make(map[string]string) params["RegionId"] = self.RegionId - params["PageSize"] = fmt.Sprintf("%d", limit) - params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + params["PageSize"] = "50" + pageNum := 1 + params["PageNumber"] = fmt.Sprintf("%d", pageNum) if len(instanceId) > 0 { params["InstanceId"] = instanceId @@ -94,24 +92,33 @@ func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, if len(category) > 0 { params["Category"] = category } - if diskIds != nil && len(diskIds) > 0 { + if len(diskIds) > 0 { params["DiskIds"] = jsonutils.Marshal(diskIds).String() } - - body, err := self.ecsRequest("DescribeDisks", params) - if err != nil { - log.Errorf("GetDisks fail %s", err) - return nil, 0, err + if len(snapshotpolicyId) > 0 { + params["AutoSnapshotPolicyId"] = snapshotpolicyId } + ret := []SDisk{} + for { + body, err := self.ecsRequest("DescribeDisks", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeDisks") + } - disks := make([]SDisk, 0) - err = body.Unmarshal(&disks, "Disks", "Disk") - if err != nil { - log.Errorf("Unmarshal disk details fail %s", err) - return nil, 0, err + part := make([]SDisk, 0) + err = body.Unmarshal(&part, "Disks", "Disk") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + ret = append(ret, part...) + total, _ := body.Int("TotalCount") + if len(ret) >= int(total) || len(part) == 0 { + break + } + pageNum++ + params["PageNumber"] = fmt.Sprintf("%d", pageNum) } - total, _ := body.Int("TotalCount") - return disks, int(total), nil + return ret, nil } func (self *SDisk) GetId() string { @@ -272,14 +279,16 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz } func (self *SRegion) getDisk(diskId string) (*SDisk, error) { - disks, total, err := self.GetDisks("", "", "", []string{diskId}, 0, 1) + disks, err := self.GetDisks("", "", "", []string{diskId}, "") if err != nil { return nil, err } - if total != 1 { - return nil, cloudprovider.ErrNotFound + for i := range disks { + if disks[i].DiskId == diskId { + return &disks[i], nil + } } - return &disks[0], nil + return nil, errors.Wrapf(cloudprovider.ErrNotFound, diskId) } func (self *SRegion) DeleteDisk(diskId string) error { @@ -399,13 +408,6 @@ func (self *SDisk) GetCreatedAt() time.Time { return self.CreationTime } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - if len(self.AutoSnapshotPolicyId) == 0 { - return []string{}, nil - } - return []string{self.AutoSnapshotPolicyId}, nil -} - func (self *SDisk) GetExpiredAt() time.Time { return convertExpiredAt(self.ExpiredTime) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/instance.go index 33810ef65e1..f6fb0a85fd4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/instance.go @@ -238,15 +238,9 @@ func (a byAttachedTime) Less(i, j int) bool { func (self *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { disks := []SDisk{} - for { - part, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, len(disks), 50) - if err != nil { - return nil, errors.Wrapf(err, "GetDisks for %s", self.InstanceId) - } - disks = append(disks, part...) - if len(disks) >= total { - break - } + disks, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, "") + if err != nil { + return nil, errors.Wrapf(err, "GetDisks for %s", self.InstanceId) } sort.Sort(byAttachedTime(disks)) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/snapshot_policy.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/snapshot_policy.go index 92cab8809e0..373eaae7750 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/snapshot_policy.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/snapshot_policy.go @@ -19,9 +19,10 @@ import ( "sort" "strconv" + "github.com/pkg/errors" "yunion.io/x/jsonutils" - api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/apis" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud" ) @@ -57,13 +58,13 @@ func (self *SSnapshotPolicy) GetName() string { } func (self *SSnapshotPolicy) GetStatus() string { - // XXX: apsara文档与实际返回值不符 - if self.Status == Normal || self.Status == Available { - return api.SNAPSHOT_POLICY_READY - } else if self.Status == Creating { - return api.SNAPSHOT_POLICY_CREATING - } else { - return api.SNAPSHOT_POLICY_UNKNOWN + switch self.Status { + case Normal, Available: + return apis.STATUS_AVAILABLE + case Creating: + return apis.STATUS_CREATING + default: + return apis.STATUS_UNKNOWN } } @@ -131,10 +132,6 @@ func (self *SSnapshotPolicy) GetTimePoints() ([]int, error) { return parsePolicy(self.TimePoints) } -func (self *SSnapshotPolicy) IsActivated() bool { - return true -} - func (self *SRegion) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) { snapshotPolicies, total, err := self.GetSnapshotPolicies("", 0, 50) if err != nil { @@ -222,12 +219,12 @@ func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp params["repeatWeekdays"] = jsonutils.Marshal(input.GetStringArrayRepeatWeekdays()).String() params["timePoints"] = jsonutils.Marshal(input.GetStringArrayTimePoints()).String() params["retentionDays"] = strconv.Itoa(input.RetentionDays) - params["autoSnapshotPolicyName"] = input.PolicyName - if body, err := self.ecsRequest("CreateAutoSnapshotPolicy", params); err != nil { - return "", fmt.Errorf("CreateAutoSnapshotPolicy fail %s", err) - } else { - return body.GetString("AutoSnapshotPolicyId") + params["autoSnapshotPolicyName"] = input.Name + body, err := self.ecsRequest("CreateAutoSnapshotPolicy", params) + if err != nil { + return "", errors.Wrapf(err, "CreateAutoSnapshotPolicy") } + return body.GetString("AutoSnapshotPolicyId") } func (self *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput, snapshotPolicyId string) error { @@ -249,36 +246,10 @@ func (self *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp return nil } -//func (self *SRegion) UpdateSnapshotPolicy( -// snapshotPolicyId string, retentionDays *int, -// repeatWeekdays, timePoints *jsonutils.JSONArray, policyName string, -//) error { -// params := make(map[string]string) -// params["RegionId"] = self.RegionId -// if len(policyName) > 0 { -// params["autoSnapshotPolicyName"] = policyName -// } -// if retentionDays != nil { -// params["retentionDays"] = strconv.Itoa(*retentionDays) -// } -// if repeatWeekdays != nil { -// params["repeatWeekdays"] = repeatWeekdays.String() -// } -// if timePoints != nil { -// params["timePoints"] = timePoints.String() -// } -// _, err := self.ecsRequest("ModifyAutoSnapshotPolicyEx", params) -// if err != nil { -// return fmt.Errorf("ModifyAutoSnapshotPolicyEx Fail %s", err) -// } -// return nil -//} - -func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["RegionId"] = self.RegionId params["autoSnapshotPolicyId"] = snapshotPolicyId - diskIds := []string{diskId} params["diskIds"] = jsonutils.Marshal(diskIds).String() _, err := self.ecsRequest("ApplyAutoSnapshotPolicy", params) if err != nil { @@ -287,10 +258,17 @@ func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId return nil } -func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SSnapshotPolicy) ApplyDisks(ids []string) error { + return self.region.ApplySnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} + +func (self *SSnapshotPolicy) GetApplyDiskIds() ([]string, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["RegionId"] = self.RegionId - diskIds := []string{diskId} params["diskIds"] = jsonutils.Marshal(diskIds).String() _, err := self.ecsRequest("CancelAutoSnapshotPolicy", params) if err != nil { @@ -298,3 +276,11 @@ func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId } return nil } + +func (self *SSnapshotPolicy) CancelDisks(ids []string) error { + return self.region.CancelSnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} + +func (self *SSnapshotPolicy) Update(opts *cloudprovider.SnapshotPolicyInput) error { + return self.region.UpdateSnapshotPolicy(opts, self.AutoSnapshotPolicyId) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/storage.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/storage.go index c8970e6729a..66a13aaa0bd 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/storage.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/storage.go @@ -72,16 +72,10 @@ func (self *SStorage) getDisks() ([]SDisk, error) { if len(self.zone.disks) > 0 { return self.zone.disks, nil } - self.zone.disks = []SDisk{} - for { - part, total, err := self.zone.region.GetDisks("", self.zone.GetId(), "", nil, len(self.zone.disks), 100) - if err != nil { - return nil, errors.Wrapf(err, "GetDisks") - } - self.zone.disks = append(self.zone.disks, part...) - if len(self.zone.disks) >= total || len(part) == 0 { - break - } + var err error + self.zone.disks, err = self.zone.region.GetDisks("", self.zone.GetId(), "", nil, "") + if err != nil { + return nil, errors.Wrapf(err, "GetDisks") } return self.zone.disks, nil } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/bingocloud/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/bingocloud/disk.go index 69ac7e92275..76be449df1f 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/bingocloud/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/bingocloud/disk.go @@ -171,10 +171,6 @@ func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string return self.storage.cluster.region.GetISnapshotById(snapshotId) } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return []string{}, nil -} - func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return cloudprovider.ErrNotImplemented } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/disk.go index 85832f78ac8..3c4494e2799 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/disk.go @@ -197,10 +197,6 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return []cloudprovider.ICloudSnapshot{}, nil } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return []string{}, nil -} - func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return self.storage.zone.region.ResizeDisk(self.GetId(), newSizeMB/1024) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go index a2641e9c452..04d44245798 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go @@ -294,34 +294,6 @@ func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSn return nil, cloudprovider.ErrNotFound } -func (self *SRegion) CreateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput) (string, error) { - return "", cloudprovider.ErrNotImplemented -} - -func (self *SRegion) UpdateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput, string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SRegion) DeleteSnapshotPolicy(policyId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { - return cloudprovider.ErrNotImplemented -} - -func (self *SRegion) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) { - return nil, cloudprovider.ErrNotImplemented -} - -func (self *SRegion) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) { - return nil, cloudprovider.ErrNotFound -} - func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { iHosts := make([]cloudprovider.ICloudHost, 0) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/disk_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/disk_base.go index f197c75a218..a526056d405 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/disk_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/disk_base.go @@ -23,10 +23,6 @@ type SDisk struct { SBillingBase } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return []string{""}, nil -} - func (self *SDisk) GetIStorageId() string { return "" } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ecloud/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ecloud/disk.go index d9ca17db97e..df5aa4f8d4f 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ecloud/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ecloud/disk.go @@ -238,10 +238,6 @@ func (s *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (s *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return []string{}, nil -} - func (s *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return cloudprovider.ErrNotImplemented } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go index e7c7764f69e..26fcacc9eeb 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go @@ -191,15 +191,6 @@ func (disk *SDisk) GetISnapshot(id string) (cloudprovider.ICloudSnapshot, error) return disk.storage.zone.region.GetSnapshot(id) } -func (disk *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - result := []string{} - for _, policy := range disk.ResourcePolicies { - globalId := strings.TrimPrefix(policy, fmt.Sprintf("%s/%s/", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION)) - result = append(result, globalId) - } - return result, nil -} - func (disk *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return disk.storage.zone.region.ResizeDisk(disk.SelfLink, int(newSizeMB>>10)) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/google.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/google.go index 0fb70a6f6dd..dc56e21eb72 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/google.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/google.go @@ -1011,6 +1011,7 @@ func (self *SGoogleClient) GetCapabilities() []string { // cloudprovider.CLOUD_CAPABILITY_EVENT, cloudprovider.CLOUD_CAPABILITY_CLOUDID, cloudprovider.CLOUD_CAPABILITY_QUOTA + cloudprovider.READ_ONLY_SUFFIX, + cloudprovider.CLOUD_CAPABILITY_SNAPSHOT_POLICY + cloudprovider.READ_ONLY_SUFFIX, } return caps } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/resourcepolicy.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/resourcepolicy.go index 28ed77b4758..f6293469930 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/resourcepolicy.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/resourcepolicy.go @@ -21,10 +21,10 @@ import ( "time" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" - api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/apis" + "yunion.io/x/cloudmux/pkg/cloudprovider" ) type SDailySchedule struct { @@ -98,10 +98,9 @@ func (region *SRegion) GetResourcePolicy(id string) (*SResourcePolicy, error) { func (policy *SResourcePolicy) GetStatus() string { switch policy.Status { case "READY": - return api.SNAPSHOT_POLICY_READY + return apis.STATUS_AVAILABLE default: - log.Errorf("unknown policy status %s", policy.Status) - return api.SNAPSHOT_POLICY_UNKNOWN + return apis.STATUS_UNKNOWN } } @@ -129,6 +128,22 @@ func (policy *SResourcePolicy) GetRetentionDays() int { return policy.SnapshotSchedulePolicy.RetentionPolicy.MaxRetentionDays } +func (policy *SResourcePolicy) ApplyDisks(ids []string) error { + return cloudprovider.ErrNotImplemented +} + +func (policy *SResourcePolicy) GetApplyDiskIds() ([]string, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (policy *SResourcePolicy) CancelDisks(ids []string) error { + return cloudprovider.ErrNotImplemented +} + +func (policy *SResourcePolicy) Delete() error { + return cloudprovider.ErrNotImplemented +} + func (policy *SResourcePolicy) GetRepeatWeekdays() ([]int, error) { result := []int{1, 2, 3, 4, 5, 6, 7} if len(policy.SnapshotSchedulePolicy.Schedule.WeeklySchedule.DayOfWeeks) > 0 { @@ -157,7 +172,3 @@ func (policy *SResourcePolicy) GetTimePoints() ([]int, error) { } return result, nil } - -func (policy *SResourcePolicy) IsActivated() bool { - return true -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/jdcloud/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/jdcloud/disk.go index f4df78587fd..9651838f7e6 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/jdcloud/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/jdcloud/disk.go @@ -203,10 +203,6 @@ func (s *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (s *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return []string{}, nil -} - func (s *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return cloudprovider.ErrNotImplemented } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go index 08f54c15f78..a41ac447899 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go @@ -274,10 +274,6 @@ func (cli *SObjectStoreClient) CreateSnapshotPolicy(*cloudprovider.SnapshotPolic return "", cloudprovider.ErrNotSupported } -func (cli *SObjectStoreClient) DeleteSnapshotPolicy(string) error { - return cloudprovider.ErrNotSupported -} - func (cli *SObjectStoreClient) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { return cloudprovider.ErrNotSupported } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/boot-disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/boot-disk.go index 8dbaf4295ae..2dd0f91076f 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/boot-disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/boot-disk.go @@ -166,10 +166,6 @@ func (self *SBootDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotImplemented } -func (self *SBootDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return nil, cloudprovider.ErrNotImplemented -} - func (self *SBootDisk) GetTemplateId() string { return self.ImageId } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/disk.go index c670198016e..af91b62f2ad 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/oracle/disk.go @@ -168,10 +168,6 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotImplemented } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return nil, cloudprovider.ErrNotImplemented -} - func (self *SDisk) GetTemplateId() string { return "" } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/disk.go index 5de94c0d084..49105a06428 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/disk.go @@ -342,10 +342,6 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return self.storage.zone.region.GetSnapshotIdByDiskId(self.GetId()) -} - func (self *SDisk) GetTemplateId() string { //return self.ImageId return "" diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go index 73bd3502e26..9704a4cdf4e 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go @@ -1019,6 +1019,7 @@ func (self *SQcloudClient) GetCapabilities() []string { cloudprovider.CLOUD_CAPABILITY_CDN + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_CONTAINER + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_CERT, + cloudprovider.CLOUD_CAPABILITY_SNAPSHOT_POLICY, } return caps } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/snapshot_policy.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/snapshot_policy.go index a5e20646dd3..ca9a36daeb6 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/snapshot_policy.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/snapshot_policy.go @@ -21,7 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" - api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/apis" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud" ) @@ -48,6 +48,7 @@ type SSnapshotPolicy struct { Policy []SSnapshotDatePolicy Activated bool `json:"IsActivated"` IsPermanent bool + DiskIdSet []string } func (self *SSnapshotPolicy) GetId() string { @@ -63,10 +64,12 @@ func (self *SSnapshotPolicy) GetGlobalId() string { } func (self *SSnapshotPolicy) GetStatus() string { - if self.AutoSnapshotPolicyState == NORMAL { - return api.SNAPSHOT_POLICY_READY + switch self.AutoSnapshotPolicyState { + case NORMAL: + return apis.STATUS_AVAILABLE + default: + return apis.STATUS_UNKNOWN } - return api.SNAPSHOT_POLICY_UNKNOWN } func (self *SSnapshotPolicy) Refresh() error { @@ -115,10 +118,6 @@ func (self *SSnapshotPolicy) GetTimePoints() ([]int, error) { return self.Policy[0].Hour, nil } -func (self *SSnapshotPolicy) IsActivated() bool { - return self.Activated -} - func (self *SRegion) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) { snapshotPolicies, total, err := self.GetSnapshotPolicies("", 0, 50) if err != nil { @@ -191,14 +190,7 @@ func (self *SRegion) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovi } func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput) (string, error) { - if input.RepeatWeekdays == nil { - return "", fmt.Errorf("Can't create snapshot policy with nil repeatWeekdays") - } - if input.TimePoints == nil { - return "", fmt.Errorf("Can't create snapshot policy with nil timePoints") - } params := make(map[string]string) - // In Qcloud, that IsPermanent is true means that keep snapshot forever, // In OneCloud, that RetentionDays is -1 means that keep snapshot forever. if input.RetentionDays == -1 { @@ -213,8 +205,8 @@ func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp } params[dayOfWeekPrefix+strconv.Itoa(index)] = strconv.Itoa(day) } - if len(input.PolicyName) > 0 { - params["AutoSnapshotPolicyName"] = input.PolicyName + if len(input.Name) > 0 { + params["AutoSnapshotPolicyName"] = input.Name } for index, hour := range input.TimePoints { params[hourPrefix+strconv.Itoa(index)] = strconv.Itoa(hour) @@ -227,33 +219,44 @@ func (self *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInp return id, nil } -func (self *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput, snapshotPolicyId string) error { - // not implement - return nil -} - -func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["AutoSnapshotPolicyId"] = snapshotPolicyId - params["DiskIds.0"] = diskId + for i, id := range diskIds { + params[fmt.Sprintf("DiskIds.%d", i)] = id + } _, err := self.cbsRequest("BindAutoSnapshotPolicy", params) if err != nil { - return errors.Wrapf(err, "Bind AutoSnapshotPolicy %s to Disk failed", snapshotPolicyId) + return errors.Wrapf(err, "BindAutoSnapshotPolicy") } return nil } -func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { +func (self *SSnapshotPolicy) ApplyDisks(ids []string) error { + return self.region.ApplySnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} + +func (self *SSnapshotPolicy) GetApplyDiskIds() ([]string, error) { + return self.DiskIdSet, nil +} + +func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error { params := make(map[string]string) params["AutoSnapshotPolicyId"] = snapshotPolicyId - params["DiskIds.0"] = diskId + for i, id := range diskIds { + params[fmt.Sprintf("DiskIds.%d", i)] = id + } _, err := self.cbsRequest("UnbindAutoSnapshotPolicy", params) if err != nil { - return errors.Wrapf(err, "Unbind AutoSnapshotPolicy %s of Disk failed", snapshotPolicyId) + return errors.Wrapf(err, "UnbindAutoSnapshotPolicy") } return nil } +func (self *SSnapshotPolicy) CancelDisks(ids []string) error { + return self.region.CancelSnapshotPolicyToDisks(self.AutoSnapshotPolicyId, ids) +} + func (self *SRegion) GetSnapshotIdByDiskId(diskID string) ([]string, error) { params := make(map[string]string) params["DiskId"] = diskID diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go index 77ea6ec782c..15ad62b8712 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go @@ -79,10 +79,6 @@ func (r *SRegion) CreateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput) return "", errors.Wrapf(cloudprovider.ErrNotImplemented, "CreateSnapshotPolicy") } -func (r *SRegion) UpdateSnapshotPolicy(input *cloudprovider.SnapshotPolicyInput, snapshotPolicyId string) error { - return errors.Wrapf(cloudprovider.ErrNotImplemented, "UpdateSnapshotPolicy") -} - func (r *SRegion) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshotPolicyById") } @@ -91,18 +87,6 @@ func (self *SRegion) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolic return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshotPolicies") } -func (self *SRegion) DeleteSnapshotPolicy(string) error { - return errors.Wrapf(cloudprovider.ErrNotImplemented, "DeleteSnapshotPolicy") -} - -func (self *SRegion) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { - return errors.Wrapf(cloudprovider.ErrNotImplemented, "ApplySnapshotPolicyToDisks") -} - -func (self *SRegion) CancelSnapshotPolicyToDisks(snapshotPolicyId string, diskId string) error { - return errors.Wrapf(cloudprovider.ErrNotImplemented, "CancelSnapshotPolicyToDisks") -} - func (self *SRegion) GetISkus() ([]cloudprovider.ICloudSku, error) { return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "GetISkus") } @@ -411,4 +395,4 @@ func (self *SRegion) GetIMiscResources() ([]cloudprovider.ICloudMiscResource, er func (self *SRegion) GetISSLCertificates() ([]cloudprovider.ICloudSSLCertificate, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISSLCertificate") -} \ No newline at end of file +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/disk.go index f43371fe1e4..fa9335e9760 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/disk.go @@ -119,10 +119,6 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotSupported } -func (self *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return nil, cloudprovider.ErrNotSupported -} - func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error { return cloudprovider.ErrNotSupported } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go index 86a76396422..7d0d7363592 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go @@ -168,10 +168,6 @@ func (disk *SDisk) GetCreatedAt() time.Time { return disk.CreatedAt } -func (disk *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { - return nil, errors.ErrNotImplemented -} - func (disk *SDisk) GetExpiredAt() time.Time { return convertExpiredAt(disk.ExpiredTime) }