diff --git a/pkg/apis/compute/storage.go b/pkg/apis/compute/storage.go index 10ca9565161..34e09694099 100644 --- a/pkg/apis/compute/storage.go +++ b/pkg/apis/compute/storage.go @@ -113,6 +113,8 @@ type StorageCreateInput struct { // swagger:ignore HardwareInfo *StorageHardwareInfo `json:"hardware_info"` + // CLVM VG Name + CLVMVgName string } type RbdTimeoutInput struct { diff --git a/pkg/apis/compute/storage_const.go b/pkg/apis/compute/storage_const.go index b605823f2c7..ced8e0c373c 100644 --- a/pkg/apis/compute/storage_const.go +++ b/pkg/apis/compute/storage_const.go @@ -34,6 +34,7 @@ const ( STORAGE_NVME_PT = "nvme_pt" // nvme passthrough STORAGE_NVME = "nvme" // nvme sriov STORAGE_LVM = "lvm" + STORAGE_CLVM = "clvm" // clustered lvm STORAGE_PUBLIC_CLOUD = compute.STORAGE_PUBLIC_CLOUD STORAGE_CLOUD_EFFICIENCY = compute.STORAGE_CLOUD_EFFICIENCY @@ -167,7 +168,7 @@ var ( STORAGE_OPENSTACK_ISCSI, STORAGE_UCLOUD_CLOUD_NORMAL, STORAGE_UCLOUD_CLOUD_SSD, STORAGE_UCLOUD_LOCAL_NORMAL, STORAGE_UCLOUD_LOCAL_SSD, STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS, STORAGE_CIFS, - STORAGE_NVME_PT, STORAGE_NVME, STORAGE_LVM, + STORAGE_NVME_PT, STORAGE_NVME, STORAGE_LVM, STORAGE_CLVM, } HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA} @@ -177,8 +178,8 @@ var ( SHARED_FILE_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS} FIEL_STORAGE = []string{STORAGE_LOCAL, STORAGE_NFS, STORAGE_GPFS} - // 目前来说只支持这些 - SHARED_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS, STORAGE_RBD} + // supported shared storage types + SHARED_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS, STORAGE_RBD, STORAGE_CLVM} ) func IsDiskTypeMatch(t1, t2 string) bool { diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 317b74fee3b..e348c529329 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -59,6 +59,51 @@ func (self *SKVMHostDriver) GetHypervisor() string { return api.HYPERVISOR_KVM } +func (self *SKVMHostDriver) validateGPFS(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, input api.HostStorageCreateInput) (api.HostStorageCreateInput, error) { + header := http.Header{} + header.Set(mcclient.AUTH_TOKEN, userCred.GetTokenString()) + header.Set(mcclient.REGION_VERSION, "v2") + params := jsonutils.NewDict() + params.Set("mount_point", jsonutils.NewString(input.MountPoint)) + urlStr := fmt.Sprintf("%s/storages/is-mount-point?%s", host.ManagerUri, params.QueryString()) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false) + if err != nil { + return input, err + } + if !jsonutils.QueryBoolean(res, "is_mount_point", false) { + return input, httperrors.NewBadRequestError("%s is not mount point %s", input.MountPoint, res) + } + urlStr = fmt.Sprintf("%s/storages/is-local-mount-point?%s", host.ManagerUri, params.QueryString()) + _, res, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false) + if err != nil { + return input, err + } + if jsonutils.QueryBoolean(res, "is_local_mount_point", false) { + return input, httperrors.NewBadRequestError("%s is local storage mount point", input.MountPoint) + } + return input, nil +} + +func (self *SKVMHostDriver) validateCLVM(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, input api.HostStorageCreateInput) (api.HostStorageCreateInput, error) { + vgName, _ := storage.StorageConf.GetString("clvm_vg_name") + if vgName == "" { + return input, httperrors.NewInternalServerError("storage has no clvm_vg_name") + } + input.MountPoint = vgName + + header := http.Header{} + header.Set(mcclient.AUTH_TOKEN, userCred.GetTokenString()) + header.Set(mcclient.REGION_VERSION, "v2") + params := jsonutils.NewDict() + params.Set("vg_name", jsonutils.NewString(input.MountPoint)) + urlStr := fmt.Sprintf("%s/storages/is-vg-exist?%s", host.ManagerUri, params.QueryString()) + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false) + if err != nil { + return input, err + } + return input, nil +} + func (self *SKVMHostDriver) ValidateAttachStorage(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, input api.HostStorageCreateInput) (api.HostStorageCreateInput, error) { if !utils.IsInStringArray(storage.StorageType, append([]string{api.STORAGE_LOCAL, api.STORAGE_NVME_PT, api.STORAGE_NVME, api.STORAGE_LVM}, api.SHARED_STORAGE...)) { return input, httperrors.NewUnsupportOperationError("Unsupport attach %s storage for %s host", storage.StorageType, host.HostType) @@ -84,28 +129,10 @@ func (self *SKVMHostDriver) ValidateAttachStorage(ctx context.Context, userCred return input, httperrors.NewInvalidStatusError("Attach nfs storage require host status is online") } if storage.StorageType == api.STORAGE_GPFS { - header := http.Header{} - header.Set(mcclient.AUTH_TOKEN, userCred.GetTokenString()) - header.Set(mcclient.REGION_VERSION, "v2") - params := jsonutils.NewDict() - params.Set("mount_point", jsonutils.NewString(input.MountPoint)) - urlStr := fmt.Sprintf("%s/storages/is-mount-point?%s", host.ManagerUri, params.QueryString()) - _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false) - if err != nil { - return input, err - } - if !jsonutils.QueryBoolean(res, "is_mount_point", false) { - return input, httperrors.NewBadRequestError("%s is not mount point %s", input.MountPoint, res) - } - urlStr = fmt.Sprintf("%s/storages/is-local-mount-point?%s", host.ManagerUri, params.QueryString()) - _, res, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false) - if err != nil { - return input, err - } - if jsonutils.QueryBoolean(res, "is_local_mount_point", false) { - return input, httperrors.NewBadRequestError("%s is local storage mount point", input.MountPoint) - } + return self.validateGPFS(ctx, userCred, host, input) } + } else if storage.StorageType == api.STORAGE_CLVM { + return self.validateCLVM(ctx, userCred, host, storage, input) } return input, nil } diff --git a/pkg/compute/storagedrivers/clvm.go b/pkg/compute/storagedrivers/clvm.go new file mode 100644 index 00000000000..1120313bca5 --- /dev/null +++ b/pkg/compute/storagedrivers/clvm.go @@ -0,0 +1,80 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagedrivers + +import ( + "context" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SCLVMStorageDriver struct { + SBaseStorageDriver +} + +func init() { + driver := SCLVMStorageDriver{} + models.RegisterStorageDriver(&driver) +} + +func (s *SCLVMStorageDriver) GetStorageType() string { + return api.STORAGE_CLVM +} + +func (s *SCLVMStorageDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.StorageCreateInput) error { + input.CLVMVgName = strings.TrimSpace(input.CLVMVgName) + if len(input.CLVMVgName) == 0 { + return httperrors.NewMissingParameterError("clvm_vg_name") + } + input.StorageConf = jsonutils.NewDict() + input.StorageConf.Set("clvm_vg_name", jsonutils.NewString(input.CLVMVgName)) + return nil +} + +func (self *SCLVMStorageDriver) ValidateSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot) error { + return nil +} + +func (s *SCLVMStorageDriver) ValidateCreateSnapshotData(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, input *api.SnapshotCreateInput) error { + return errors.Errorf("lvm storage unsupported create snapshot") +} + +func (s *SCLVMStorageDriver) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, storage *models.SStorage, data jsonutils.JSONObject) { + sc := &models.SStoragecache{} + sc.ExternalId = storage.Id + sc.Name = "clvm-" + storage.Name + time.Now().Format("2006-01-02 15:04:05") + if err := models.StoragecacheManager.TableSpec().Insert(ctx, sc); err != nil { + log.Errorf("insert storagecache for storage %s error: %v", storage.Name, err) + return + } + _, err := db.Update(storage, func() error { + storage.StoragecacheId = sc.Id + storage.Status = api.STORAGE_ONLINE + return nil + }) + if err != nil { + log.Errorf("update storagecache info for storage %s error: %v", storage.Name, err) + } +} diff --git a/pkg/compute/storagedrivers/local.go b/pkg/compute/storagedrivers/local.go index 42e32d13a77..310f386a64b 100644 --- a/pkg/compute/storagedrivers/local.go +++ b/pkg/compute/storagedrivers/local.go @@ -76,6 +76,10 @@ func (self *SLVMStorageDriver) ValidateCreateSnapshotData(ctx context.Context, u return errors.Errorf("lvm storage unsupported create snapshot") } +func (self *SLVMStorageDriver) ValidateSnapshotDelete(ctx context.Context, snapshot *models.SSnapshot) error { + return nil +} + type SNVMEPassthroughStorageDriver struct { SBaseStorageDriver } diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 07bc88f1e51..a5f1065babd 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -57,6 +57,7 @@ type SStorageManager struct { // AgentStorageImagecacheManager IImageCacheManger LVMStorageImagecacheManagers map[string]IImageCacheManger + CLVMStorageImagecacheManagers map[string]IImageCacheManger RbdStorageImagecacheManagers map[string]IImageCacheManger SharedFileStorageImagecacheManagers map[string]IImageCacheManger } @@ -91,8 +92,8 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) { } } - for i, d := range options.HostOptions.LVMVolumeGroups { - s := NewLVMStorage(ret, d, i) + for _, d := range options.HostOptions.LVMVolumeGroups { + s := NewLVMStorage(ret, d) if err := s.Accessible(); err == nil { ret.Storages = append(ret.Storages, s) if allFull && s.GetFreeSizeMb() > MINIMAL_FREE_SPACE { @@ -134,6 +135,8 @@ func (s *SStorageManager) Remove(storage IStorage) { delete(s.RbdStorageImagecacheManagers, storage.GetStoragecacheId()) } else if storage.StorageType() == api.STORAGE_LVM { delete(s.LVMStorageImagecacheManagers, storage.GetStoragecacheId()) + } else if storage.StorageType() == api.STORAGE_CLVM { + delete(s.CLVMStorageImagecacheManagers, storage.GetStoragecacheId()) } for index, iS := range s.Storages { if iS.GetId() == storage.GetId() { @@ -298,6 +301,10 @@ func (s *SStorageManager) GetStoragecacheById(scId string) IImageCacheManger { if sc, ok := s.LVMStorageImagecacheManagers[scId]; ok { return sc } + if sc, ok := s.CLVMStorageImagecacheManagers[scId]; ok { + return sc + } + return nil } @@ -309,9 +316,13 @@ func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheI if utils.IsInStringArray(storageType, api.SHARED_FILE_STORAGE) { s.InitSharedFileStorageImagecache(storagecacheId, imagecachePath) } else if storageType == api.STORAGE_RBD { - if rbdStorage := s.GetStoragecacheById(storagecacheId); rbdStorage == nil { + if rbdStorageCache := s.GetStoragecacheById(storagecacheId); rbdStorageCache == nil { s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId) } + } else if storageType == api.STORAGE_CLVM { + if clvmStorageCache := s.GetStoragecacheById(storagecacheId); clvmStorageCache == nil { + s.AddCLVMStorageImagecache(storage.GetPath(), storage, storagecacheId) + } } } @@ -339,6 +350,16 @@ func (s *SStorageManager) InitSharedFileStorageImagecache(storagecacheId, path s } } +func (s *SStorageManager) AddCLVMStorageImagecache(imagecachePath string, storage IStorage, storagecacheId string) { + if s.CLVMStorageImagecacheManagers == nil { + s.CLVMStorageImagecacheManagers = map[string]IImageCacheManger{} + } + if _, ok := s.RbdStorageImagecacheManagers[storagecacheId]; !ok { + imagecache := NewLVMImageCacheManager(s, imagecachePath, storagecacheId) + s.CLVMStorageImagecacheManagers[storagecacheId] = imagecache + } +} + func (s *SStorageManager) AddRbdStorageImagecache(imagecachePath string, storage IStorage, storagecacheId string) { if s.RbdStorageImagecacheManagers == nil { s.RbdStorageImagecacheManagers = map[string]IImageCacheManger{} diff --git a/pkg/hostman/storageman/disk_clvm.go b/pkg/hostman/storageman/disk_clvm.go new file mode 100644 index 00000000000..ef97f9a96e2 --- /dev/null +++ b/pkg/hostman/storageman/disk_clvm.go @@ -0,0 +1,275 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageman + +import ( + "context" + "fmt" + "path" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/appctx" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/qemuimgfmt" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" + "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" + "yunion.io/x/onecloud/pkg/hostman/storageman/storageutils" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/qemuimg" +) + +type SCLVMDisk struct { + SBaseDisk +} + +func (d *SCLVMDisk) DiskBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { + return nil, errors.ErrNotImplemented +} + +func NewCLVMDisk(storage IStorage, id string) *SCLVMDisk { + return &SCLVMDisk{ + SBaseDisk: *NewBaseDisk(storage, id), + } +} + +func (d *SCLVMDisk) GetType() string { + return api.STORAGE_CLVM +} + +// /dev// +func (d *SCLVMDisk) GetLvPath() string { + return path.Join("/dev", d.Storage.GetPath(), d.Id) +} + +func (d *SCLVMDisk) GetPath() string { + return path.Join("/dev", d.Storage.GetPath(), d.Id) +} + +func (d *SCLVMDisk) GetDiskSetupScripts(idx int) string { + return fmt.Sprintf("DISK_%d='%s'\n", idx, d.GetPath()) +} + +func (d *SCLVMDisk) GetDiskDesc() jsonutils.JSONObject { + qemuImg, err := qemuimg.NewQemuImage(d.GetPath()) + if err != nil { + log.Errorln(err) + return nil + } + + var desc = jsonutils.NewDict() + desc.Set("disk_id", jsonutils.NewString(d.Id)) + desc.Set("disk_size", jsonutils.NewInt(qemuImg.SizeBytes/1024/1024)) + desc.Set("format", jsonutils.NewString(string(qemuImg.Format))) + desc.Set("disk_path", jsonutils.NewString(d.Storage.GetPath())) + return desc +} + +func (d *SCLVMDisk) CreateRaw( + ctx context.Context, sizeMb int, diskFormat string, fsFormat string, + encryptInfo *apis.SEncryptInfo, diskId string, back string, +) (jsonutils.JSONObject, error) { + if fileutils2.Exists(d.GetPath()) { + if err := lvmutils.LvRemove(d.GetLvPath()); err != nil { + return nil, errors.Wrap(err, "CreateRaw lvremove") + } + } + if err := lvmutils.LvCreate(d.Storage.GetPath(), d.Id, int64(sizeMb)*1024*1024); err != nil { + return nil, errors.Wrap(err, "CreateRaw") + } + + diskInfo := &deployapi.DiskInfo{ + Path: d.GetPath(), + } + if utils.IsInStringArray(fsFormat, []string{"swap", "ext2", "ext3", "ext4", "xfs"}) { + d.FormatFs(fsFormat, diskId, diskInfo) + } + return d.GetDiskDesc(), nil +} + +func (d *SCLVMDisk) CreateFromTemplate( + ctx context.Context, imageId, format string, sizeMb int64, encryptInfo *apis.SEncryptInfo, +) (jsonutils.JSONObject, error) { + if fileutils2.Exists(d.GetPath()) { + if err := lvmutils.LvRemove(d.GetLvPath()); err != nil { + return nil, errors.Wrap(err, "CreateRaw lvremove") + } + } + + var imageCacheManager = storageManager.GetStoragecacheById(d.Storage.GetStoragecacheId()) + ret, err := d.createFromTemplate(ctx, imageId, format, sizeMb, imageCacheManager, encryptInfo) + if err != nil { + return nil, err + } + retSize, _ := ret.Int("disk_size") + log.Infof("REQSIZE: %d, RETSIZE: %d", sizeMb, retSize) + if sizeMb > retSize { + params := jsonutils.NewDict() + params.Set("size", jsonutils.NewInt(sizeMb)) + if encryptInfo != nil { + params.Set("encrypt_info", jsonutils.Marshal(encryptInfo)) + } + return d.Resize(ctx, params) + } + return ret, nil +} + +func (d *SCLVMDisk) createFromTemplate( + ctx context.Context, imageId, format string, sizeMb int64, imageCacheManager IImageCacheManger, encryptInfo *apis.SEncryptInfo, +) (jsonutils.JSONObject, error) { + input := api.CacheImageInput{ImageId: imageId, Zone: d.GetZoneId()} + imageCache, err := imageCacheManager.AcquireImage(ctx, input, nil) + if err != nil { + return nil, errors.Wrapf(err, "AcquireImage") + } + + defer imageCacheManager.ReleaseImage(ctx, imageId) + cacheImagePath := imageCache.GetPath() + + lvSizeMb := d.getQcow2LvSize(sizeMb) + if err := lvmutils.LvCreate(d.Storage.GetPath(), d.Id, lvSizeMb*1024*1024); err != nil { + return nil, errors.Wrap(err, "CreateRaw") + } + newImg, err := qemuimg.NewQemuImage(d.GetPath()) + if err != nil { + return nil, errors.Wrapf(err, "NewQemuImage(%s)", d.GetPath()) + } + err = newImg.CreateQcow2(int(sizeMb), false, cacheImagePath, "", "", "") + if err != nil { + return nil, errors.Wrapf(err, "CreateQcow2(%s)", cacheImagePath) + } + + return d.GetDiskDesc(), nil +} + +func (d *SCLVMDisk) Probe() error { + if !fileutils2.Exists(d.GetPath()) { + return errors.Wrapf(cloudprovider.ErrNotFound, "%s", d.GetPath()) + } + return nil +} + +func (d *SCLVMDisk) GetSnapshotDir() string { + return "" +} + +func (d *SCLVMDisk) OnRebuildRoot(ctx context.Context, params api.DiskAllocateInput) error { + _, err := d.Delete(ctx, api.DiskDeleteInput{}) + return err +} + +func (d *SCLVMDisk) Delete(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { + if err := lvmutils.LvRemove(d.GetLvPath()); err != nil { + return nil, errors.Wrap(err, "Delete lvremove") + } + d.Storage.RemoveDisk(d) + return nil, nil +} + +func (d *SCLVMDisk) getQcow2LvSize(sizeMb int64) int64 { + // Qcow2 cluster size 2M, 100G reserve 1M for qcow2 metadata + metaSize := sizeMb/1024/100 + 2 + return sizeMb + metaSize +} + +func (d *SCLVMDisk) Resize(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { + diskInfo, ok := params.(*jsonutils.JSONDict) + if !ok { + return nil, hostutils.ParamsError + } + sizeMb, _ := diskInfo.Int("size") + + qemuImg, err := qemuimg.NewQemuImage(d.GetPath()) + if err != nil { + return nil, errors.Wrap(err, "lvm qemuimg.NewQemuImage") + } + + lvsize := sizeMb + if qemuImg.Format == qemuimgfmt.QCOW2 { + lvsize = d.getQcow2LvSize(sizeMb) + } + + err = lvmutils.LvResize(d.Storage.GetPath(), d.GetPath(), lvsize*1024*1024) + if err != nil { + return nil, errors.Wrap(err, "lv resize") + } + err = qemuImg.Resize(int(sizeMb)) + if err != nil { + return nil, errors.Wrap(err, "qemuImg resize") + } + + resizeFsInfo := &deployapi.DiskInfo{ + Path: d.GetPath(), + } + if err := d.ResizeFs(resizeFsInfo); err != nil { + log.Errorf("Resize fs %s fail %s", d.GetPath(), err) + } + return d.GetDiskDesc(), nil +} + +func (d *SCLVMDisk) PrepareSaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { + if err := d.Probe(); err != nil { + return nil, err + } + destDir := d.Storage.GetImgsaveBackupPath() + if err := procutils.NewCommand("mkdir", "-p", destDir).Run(); err != nil { + log.Errorln(err) + return nil, err + } + freeSizeMb, err := storageutils.GetFreeSizeMb(destDir) + if err != nil { + return nil, errors.Wrap(err, "lvm storageutils.GetFreeSizeMb") + } + qemuImg, err := qemuimg.NewQemuImage(d.GetPath()) + if err != nil { + return nil, errors.Wrap(err, "lvm qemuimg.NewQemuImage") + } + if int(qemuImg.SizeBytes/1024/1024) >= freeSizeMb*4/5 { + return nil, errors.Errorf("image cache dir free size is not enough") + } + + backupPath := path.Join(destDir, fmt.Sprintf("%s.%s", d.Id, appctx.AppContextTaskId(ctx))) + srcInfo := qemuimg.SImageInfo{ + Path: d.GetPath(), + Format: qemuImg.Format, + IoLevel: qemuimg.IONiceNone, + Password: "", + } + destInfo := qemuimg.SImageInfo{ + Path: backupPath, + Format: qemuimgfmt.QCOW2, + IoLevel: qemuimg.IONiceNone, + Password: "", + } + if err = qemuimg.Convert(srcInfo, destInfo, true, nil); err != nil { + log.Errorln(err) + procutils.NewCommand("rm", "-f", backupPath).Run() + return nil, err + } + res := jsonutils.NewDict() + res.Set("backup", jsonutils.NewString(backupPath)) + return res, nil +} + +func (d *SCLVMDisk) IsFile() bool { + return false +} diff --git a/pkg/hostman/storageman/disk_lvm.go b/pkg/hostman/storageman/disk_lvm.go index 30258b771ae..a1685592d2b 100644 --- a/pkg/hostman/storageman/disk_lvm.go +++ b/pkg/hostman/storageman/disk_lvm.go @@ -48,6 +48,10 @@ func (d *SLVMDisk) GetSnapshotDir() string { return "" } +func (d *SLVMDisk) GetType() string { + return api.STORAGE_LVM +} + // /dev// func (d *SLVMDisk) GetLvPath() string { return path.Join("/dev", d.Storage.GetPath(), d.Id) @@ -112,7 +116,7 @@ func (d *SLVMDisk) CleanUpDisk() error { // disk path /dev// if fileutils2.Exists(d.GetLvPath()) { if err := lvmutils.LvRemove(d.GetLvPath()); err != nil { - return nil + return err } } return nil @@ -250,7 +254,7 @@ func (d *SLVMDisk) CreateFromTemplate( } var imageCacheManager = storageManager.GetStoragecacheById(d.Storage.GetStoragecacheId()) - ret, err := d.createFromTemplate(ctx, imageId, format, size*1024*1024, imageCacheManager, encryptInfo) + ret, err := d.createFromTemplate(ctx, imageId, format, size, imageCacheManager, encryptInfo) if err != nil { return nil, err } @@ -312,7 +316,7 @@ func (d *SLVMDisk) PrepareSaveToGlance(ctx context.Context, params interface{}) } func (d *SLVMDisk) createFromTemplate( - ctx context.Context, imageId, format string, size int64, imageCacheManager IImageCacheManger, encryptInfo *apis.SEncryptInfo, + ctx context.Context, imageId, format string, sizeMb int64, imageCacheManager IImageCacheManger, encryptInfo *apis.SEncryptInfo, ) (jsonutils.JSONObject, error) { input := api.CacheImageInput{ImageId: imageId, Zone: d.GetZoneId()} imageCache, err := imageCacheManager.AcquireImage(ctx, input, nil) diff --git a/pkg/hostman/storageman/imagecachemanager_lvm.go b/pkg/hostman/storageman/imagecachemanager_lvm.go index 38345cce219..9243cc5695f 100644 --- a/pkg/hostman/storageman/imagecachemanager_lvm.go +++ b/pkg/hostman/storageman/imagecachemanager_lvm.go @@ -46,6 +46,7 @@ func NewLVMImageCacheManager(manager IStorageManager, cachePath string, storagec imageCacheManager.cachePath = cachePath imageCacheManager.cachedImages = make(map[string]IImageCache, 0) + imageCacheManager.loadCache(context.Background()) return imageCacheManager } diff --git a/pkg/hostman/storageman/lvmutils/lvmutils.go b/pkg/hostman/storageman/lvmutils/lvmutils.go index d5a44a0769a..3ebd362dbf9 100644 --- a/pkg/hostman/storageman/lvmutils/lvmutils.go +++ b/pkg/hostman/storageman/lvmutils/lvmutils.go @@ -34,9 +34,8 @@ type LvNames struct { } func GetLvNames(vg string) ([]string, error) { - lvs, err := procutils.NewRemoteCommandAsFarAsPossible( - "lvm", "lvs", "--reportformat", "json", "-o", "lv_name", vg, - ).Output() + cmd := fmt.Sprintf("lvm lvs --reportformat json -o lv_name %s 2>/dev/null", vg) + lvs, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output() if err != nil { return nil, errors.Wrap(err, "lvm lvs") } @@ -64,9 +63,8 @@ type LvOrigin struct { } func GetLvOrigin(lvPath string) (string, error) { - lvs, err := procutils.NewRemoteCommandAsFarAsPossible( - "lvm", "lvs", "--reportformat", "json", "-o", "origin", lvPath, - ).Output() + cmd := fmt.Sprintf("lvm lvs --reportformat json -o origin %s 2>/dev/null", lvPath) + lvs, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output() if err != nil { return "", errors.Wrap(err, "lvm lvs") } @@ -99,9 +97,8 @@ type VgReports struct { } func GetVgProps(vg string) (*VgProps, error) { - out, err := procutils.NewRemoteCommandAsFarAsPossible( - "lvm", "vgs", "--reportformat", "json", "-o", "vg_free,vg_size,vg_extent_size", "--units=B", vg, - ).Output() + cmd := fmt.Sprintf("lvm vgs --reportformat json -o vg_free,vg_size,vg_extent_size --units=B %s 2>/dev/null", vg) + out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output() if err != nil { return nil, errors.Wrapf(err, "exec lvm command: %s", out) } @@ -212,3 +209,11 @@ $size1 $size2 linear $2 0" | dmsetup create $3 } return nil } + +func VgDisplay(vgName string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("lvm", "vgdisplay", vgName).Output() + if err != nil { + return errors.Wrapf(err, "vgdisplay %s failed %s", vgName, out) + } + return nil +} diff --git a/pkg/hostman/storageman/storage_base.go b/pkg/hostman/storageman/storage_base.go index e2fa19c6534..191013eda54 100644 --- a/pkg/hostman/storageman/storage_base.go +++ b/pkg/hostman/storageman/storage_base.go @@ -355,6 +355,7 @@ func (s *SBaseStorage) CreateDiskByDiskinfo(ctx context.Context, params interfac return nil, fmt.Errorf("Fail to Create disk %s", createParams.DiskId) } + log.Infof("storage %v start create disk", createParams.Storage.StorageType()) switch { case len(createParams.DiskInfo.SnapshotId) > 0: log.Infof("CreateDiskFromSnpashot %s", createParams) diff --git a/pkg/hostman/storageman/storage_clvm.go b/pkg/hostman/storageman/storage_clvm.go new file mode 100644 index 00000000000..ecdf9b68f9e --- /dev/null +++ b/pkg/hostman/storageman/storage_clvm.go @@ -0,0 +1,109 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageman + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" +) + +func init() { + registerStorageFactory(&SCLVMStorageFactory{}) +} + +type SCLVMStorageFactory struct { +} + +func (factory *SCLVMStorageFactory) NewStorage(manager *SStorageManager, mountPoint string) IStorage { + return NewCLVMStorage(manager, mountPoint) +} + +func (factory *SCLVMStorageFactory) StorageType() string { + return api.STORAGE_CLVM +} + +type SCLVMStorage struct { + *SLVMStorage +} + +func NewCLVMStorage(manager *SStorageManager, vgName string) *SCLVMStorage { + var ret = new(SCLVMStorage) + ret.SLVMStorage = NewLVMStorage(manager, vgName) + return ret +} + +func (s *SCLVMStorage) newDisk(diskId string) IDisk { + return NewCLVMDisk(s, diskId) +} + +func (s *SCLVMStorage) StorageType() string { + return api.STORAGE_CLVM +} + +func (s *SCLVMStorage) IsLocal() bool { + return false +} + +func (s *SCLVMStorage) CreateDisk(diskId string) IDisk { + s.DiskLock.Lock() + defer s.DiskLock.Unlock() + disk := NewCLVMDisk(s, diskId) + s.Disks = append(s.Disks, disk) + return disk +} + +func (s *SCLVMStorage) GetDiskById(diskId string) (IDisk, error) { + s.DiskLock.Lock() + defer s.DiskLock.Unlock() + for i := 0; i < len(s.Disks); i++ { + if s.Disks[i].GetId() == diskId { + err := s.Disks[i].Probe() + if err != nil { + return nil, errors.Wrapf(err, "disk.Probe") + } + return s.Disks[i], nil + } + } + + var disk = NewCLVMDisk(s, diskId) + if disk.Probe() == nil { + s.Disks = append(s.Disks, disk) + return disk, nil + } + return nil, errors.ErrNotFound +} + +func (s *SCLVMStorage) Accessible() error { + if err := lvmutils.VgDisplay(s.Path); err != nil { + return err + } + return nil +} + +func (s *SCLVMStorage) SetStorageInfo(storageId, storageName string, conf jsonutils.JSONObject) error { + s.StorageId = storageId + s.StorageName = storageName + if dconf, ok := conf.(*jsonutils.JSONDict); ok { + s.StorageConf = dconf + } + + if err := s.Accessible(); err != nil { + return err + } + return nil +} diff --git a/pkg/hostman/storageman/storage_lvm.go b/pkg/hostman/storageman/storage_lvm.go index 12517a2e0ae..82150123e2e 100644 --- a/pkg/hostman/storageman/storage_lvm.go +++ b/pkg/hostman/storageman/storage_lvm.go @@ -37,14 +37,11 @@ import ( type SLVMStorage struct { SBaseStorage - - Index int } -func NewLVMStorage(manager *SStorageManager, vgName string, index int) *SLVMStorage { +func NewLVMStorage(manager *SStorageManager, vgName string) *SLVMStorage { var ret = new(SLVMStorage) ret.SBaseStorage = *NewBaseStorage(manager, vgName) - ret.Index = index return ret } @@ -57,7 +54,7 @@ func (s *SLVMStorage) IsLocal() bool { } func (s *SLVMStorage) GetComposedName() string { - return fmt.Sprintf("host_%s_%s_storage_%d", s.Manager.host.GetMasterIp(), s.StorageType(), s.Index) + return fmt.Sprintf("host_%s_%s_storage_%s", s.Manager.host.GetMasterIp(), s.StorageType(), s.Path) } func (s *SLVMStorage) GetMediumType() (string, error) { @@ -86,7 +83,7 @@ func (s *SLVMStorage) getAvailSizeMb() (int64, error) { return -1, err } - log.Infof("LVM Storage %s sizeMb %d", s.GetPath(), vgProps.VgSize/1024/1024) + log.Debugf("LVM Storage %s sizeMb %d", s.GetPath(), vgProps.VgSize/1024/1024) return vgProps.VgSize / 1024 / 1024, nil } @@ -323,6 +320,13 @@ func (s *SLVMStorage) GetImgsaveBackupPath() string { } func (s *SLVMStorage) Accessible() error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("pvscan", "--cache").Output() + if err != nil { + return errors.Wrapf(err, "pvscan --cache failed %s", out) + } + if err := lvmutils.VgDisplay(s.Path); err != nil { + return err + } return nil } diff --git a/pkg/hostman/storageman/storagehandler/storagehandler.go b/pkg/hostman/storageman/storagehandler/storagehandler.go index d29ac6dc1bf..508aa94130d 100644 --- a/pkg/hostman/storageman/storagehandler/storagehandler.go +++ b/pkg/hostman/storageman/storagehandler/storagehandler.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage" + "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" @@ -64,6 +65,9 @@ func AddStorageHandler(prefix string, app *appsrv.Application) { app.AddHandler("GET", fmt.Sprintf("%s/%s/is-local-mount-point", prefix, keyWords), auth.Authenticate(storageIsLocalMountPoint)) + app.AddHandler("GET", + fmt.Sprintf("%s/%s/is-vg-exist", prefix, keyWords), + auth.Authenticate(storageIsVgExist)) app.AddHandler("POST", fmt.Sprintf("%s/%s/delete-backup", prefix, keyWords), auth.Authenticate(storageDeleteBackup)) @@ -108,6 +112,21 @@ func storageIsLocalMountPoint(ctx context.Context, w http.ResponseWriter, r *htt } } +func storageIsVgExist(ctx context.Context, w http.ResponseWriter, r *http.Request) { + _, query, _ := appsrv.FetchEnv(ctx, w, r) + vgName, err := query.GetString("vg_name") + if err != nil { + hostutils.Response(ctx, w, httperrors.NewMissingParameterError("vg_name")) + return + } + if err := lvmutils.VgDisplay(vgName); err != nil { + log.Errorf("vg %s display failed %s", vgName, err) + hostutils.Response(ctx, w, httperrors.NewInternalServerError(err.Error())) + return + } + hostutils.ResponseOk(ctx, w) +} + func storageVerifyMountPoint(ctx context.Context, w http.ResponseWriter, r *http.Request) { _, query, _ := appsrv.FetchEnv(ctx, w, r) mountPoint, err := query.GetString("mount_point") diff --git a/pkg/mcclient/options/compute/storage.go b/pkg/mcclient/options/compute/storage.go index 2ca68788b40..496280133bb 100644 --- a/pkg/mcclient/options/compute/storage.go +++ b/pkg/mcclient/options/compute/storage.go @@ -68,7 +68,7 @@ type StorageCreateOptions struct { ZONE string `help:"Zone id of storage"` Capacity int64 `help:"Capacity of the Storage"` MediumType string `help:"Medium type" choices:"ssd|rotate" default:"ssd"` - StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|gpfs|baremetal"` + StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|gpfs|baremetal|clvm"` RbdMonHost string `help:"Ceph mon_host config"` RbdRadosMonOpTimeout int64 `help:"ceph rados_mon_op_timeout"` RbdRadosOsdOpTimeout int64 `help:"ceph rados_osd_op_timeout"` @@ -77,6 +77,7 @@ type StorageCreateOptions struct { RbdPool string `help:"Ceph Pool Name"` NfsHost string `help:"NFS host"` NfsSharedDir string `help:"NFS shared dir"` + ClvmVgName string `help:"clvm vg name"` } func (opts *StorageCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -88,6 +89,10 @@ func (opts *StorageCreateOptions) Params() (jsonutils.JSONObject, error) { if len(opts.NfsHost) == 0 || len(opts.NfsSharedDir) == 0 { return nil, fmt.Errorf("Storage type nfs missing conf host or shared dir") } + } else if opts.StorageType == "clvm" { + if len(opts.ClvmVgName) == 0 { + return nil, fmt.Errorf("Storage type clvm missing conf clvm_vg_name") + } } return options.StructToParams(opts) } diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 1cfd3f85fdf..3c6820e462c 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -609,7 +609,7 @@ func (img *SQemuImage) CloneRaw(name string) (*SQemuImage, error) { } func (img *SQemuImage) create(sizeMB int, format qemuimgfmt.TImageFormat, options []string, extraArgs []string) error { - if img.IsValid() { + if img.IsValid() && img.Format != qemuimgfmt.RAW { return fmt.Errorf("create: the image is valid??? %s", img.Format) } args := []string{"-c", strconv.Itoa(int(img.IoLevel)),