From f46c325b5bf426cccd850b7e4223208e51ddc5d8 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 9 Jan 2025 14:04:45 +0800 Subject: [PATCH] feat(region,host,climc): support container post image overlay (#21942) --- cmd/climc/shell/image/images.go | 46 ++++++--- pkg/apis/compute/container.go | 21 +++- pkg/apis/container.go | 35 ++++++- pkg/apis/host/container.go | 1 + pkg/apis/image/consts.go | 26 ++--- .../container_drivers/volume_mount/disk.go | 69 +++++++++---- .../volume_mount/disk_pov_host_path.go | 35 +++++++ .../volume_mount/disk_pov_image.go | 70 +++++++++++++ pkg/compute/models/container_drivers.go | 8 ++ pkg/compute/models/containers.go | 64 +++++++++--- .../tasks/container_cache_images_task.go | 98 +++++++++++++++++++ .../tasks/container_volume_mount_task.go | 50 +++++++++- .../tasks/storage_batch_cache_image_task.go | 54 ++++++++++ .../container/volume_mount/disk/disk.go | 35 +++++-- .../volume_mount/disk/post_overlay.go | 76 +++++--------- .../disk/post_overlay_hostpath.go | 88 +++++++++++++++++ .../volume_mount/disk/post_overlay_image.go | 97 ++++++++++++++++++ .../container/volume_mount/interface.go | 1 + pkg/hostman/guestman/pod.go | 6 +- pkg/hostman/storageman/imagecache_local.go | 13 ++- pkg/mcclient/options/compute/containers.go | 30 ++++-- 21 files changed, 782 insertions(+), 141 deletions(-) create mode 100644 pkg/compute/container_drivers/volume_mount/disk_pov_host_path.go create mode 100644 pkg/compute/container_drivers/volume_mount/disk_pov_image.go create mode 100644 pkg/compute/tasks/container_cache_images_task.go create mode 100644 pkg/compute/tasks/storage_batch_cache_image_task.go create mode 100644 pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go create mode 100644 pkg/hostman/container/volume_mount/disk/post_overlay_image.go diff --git a/cmd/climc/shell/image/images.go b/cmd/climc/shell/image/images.go index 3e4c92291f6..d0299b94421 100644 --- a/cmd/climc/shell/image/images.go +++ b/cmd/climc/shell/image/images.go @@ -23,9 +23,11 @@ import ( "github.com/cheggaaa/pb/v3" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/printutils" "yunion.io/x/onecloud/cmd/climc/shell" + imageapi "yunion.io/x/onecloud/pkg/apis/image" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" modules "yunion.io/x/onecloud/pkg/mcclient/modules/image" @@ -62,6 +64,9 @@ type ImageOptionalOptions struct { DisableUsbKbd bool `help:"Disable usb keyboard on this image(for hypervisor kvm)"` BootMode string `help:"UEFI support" choices:"UEFI|BIOS"` VdiProtocol string `help:"VDI protocol" choices:"vnc|spice"` + // for container usage + UsedByPostOverlay bool `help:"Used by container post-overlay"` + InternalPathMap []string `help:"Internal path map, e.g. 'com.taobao.taobao/10.42.12/data/data/com.taobao.taobao:/data/data/com.taobao.taobao'"` } func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDict, args ImageOptionalOptions) error { @@ -106,50 +111,65 @@ func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDi } params.Add(jsonutils.NewString(projectId), "owner") } + keyProperties := "properties" if len(args.OsType) > 0 { - params.Add(jsonutils.NewString(args.OsType), "properties", "os_type") + params.Add(jsonutils.NewString(args.OsType), keyProperties, "os_type") } if len(args.OsDist) > 0 { - params.Add(jsonutils.NewString(args.OsDist), "properties", "os_distribution") + params.Add(jsonutils.NewString(args.OsDist), keyProperties, "os_distribution") } if len(args.OsVersion) > 0 { - params.Add(jsonutils.NewString(args.OsVersion), "properties", "os_version") + params.Add(jsonutils.NewString(args.OsVersion), keyProperties, "os_version") } if len(args.OsCodename) > 0 { - params.Add(jsonutils.NewString(args.OsCodename), "properties", "os_codename") + params.Add(jsonutils.NewString(args.OsCodename), keyProperties, "os_codename") } if len(args.OsArch) > 0 { - params.Add(jsonutils.NewString(args.OsArch), "properties", "os_arch") + params.Add(jsonutils.NewString(args.OsArch), keyProperties, "os_arch") params.Add(jsonutils.NewString(args.OsArch), "os_arch") } if len(args.OsLang) > 0 { - params.Add(jsonutils.NewString(args.OsLang), "properties", "os_language") + params.Add(jsonutils.NewString(args.OsLang), keyProperties, "os_language") } if args.Preference > 0 { params.Add(jsonutils.NewString(fmt.Sprintf("%d", args.Preference)), "properties", "preference") } if len(args.Notes) > 0 { - params.Add(jsonutils.NewString(args.Notes), "properties", "notes") + params.Add(jsonutils.NewString(args.Notes), keyProperties, "notes") } if len(args.DiskDriver) > 0 { - params.Add(jsonutils.NewString(args.DiskDriver), "properties", "disk_driver") + params.Add(jsonutils.NewString(args.DiskDriver), keyProperties, "disk_driver") } if len(args.NetDriver) > 0 { - params.Add(jsonutils.NewString(args.NetDriver), "properties", "net_driver") + params.Add(jsonutils.NewString(args.NetDriver), keyProperties, "net_driver") } if len(args.Hypervisor) > 0 { params.Add(jsonutils.NewString(strings.Join(args.Hypervisor, ",")), "properties", "hypervisor") } if args.DisableUsbKbd { - params.Add(jsonutils.NewString("true"), "properties", "disable_usb_kbd") + params.Add(jsonutils.NewString("true"), keyProperties, "disable_usb_kbd") } if args.BootMode == "UEFI" { - params.Add(jsonutils.JSONTrue, "properties", "uefi_support") + params.Add(jsonutils.JSONTrue, keyProperties, "uefi_support") } else if args.BootMode == "BIOS" { - params.Add(jsonutils.JSONFalse, "properties", "uefi_support") + params.Add(jsonutils.JSONFalse, keyProperties, "uefi_support") } if len(args.VdiProtocol) > 0 { - params.Add(jsonutils.NewString(args.VdiProtocol), "properties", "vdi_protocol") + params.Add(jsonutils.NewString(args.VdiProtocol), keyProperties, "vdi_protocol") + } + if args.UsedByPostOverlay { + params.Add(jsonutils.NewString("true"), keyProperties, imageapi.IMAGE_USED_BY_POST_OVERLAY) + } + if len(args.InternalPathMap) > 0 { + dirMap := make(map[string]string) + for _, dir := range args.InternalPathMap { + parts := strings.Split(dir, ":") + if len(parts) != 2 { + return errors.Errorf("internal dir format error: %s", dir) + } + dirMap[parts[0]] = parts[1] + } + params.Add(jsonutils.NewString(jsonutils.Marshal(dirMap).String()), keyProperties, imageapi.IMAGE_INTERNAL_PATH_MAP) } return nil } diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 8aec8d38ddd..5041783c136 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -87,6 +87,8 @@ const ( CONTAINER_STATUS_ADD_POST_OVERLY_FAILED = "add_post_overly_failed" CONTAINER_STATUS_REMOVE_POST_OVERLY = "removing_post_overly" CONTAINER_STATUS_REMOVE_POST_OVERLY_FAILED = "remove_post_overly_failed" + CONTAINER_STATUS_CACHE_IMAGE = "caching_image" + CONTAINER_STATUS_CACHE_IMAGE_FAILED = "caching_image_failed" ) var ( @@ -177,10 +179,12 @@ type ContainerDevice struct { } type ContainerSaveVolumeMountToImageInput struct { - Name string `json:"name"` - GenerateName string `json:"generate_name"` - Notes string `json:"notes"` - Index int `json:"index"` + Name string `json:"name"` + GenerateName string `json:"generate_name"` + Notes string `json:"notes"` + Index int `json:"index"` + Dirs []string `json:"dirs"` + UsedByPostOverlay bool `json:"used_by_post_overlay"` } type ContainerExecInfoOutput struct { @@ -277,3 +281,12 @@ type ContainerVolumeMountRemovePostOverlayInput struct { UseLazy bool `json:"use_lazy"` ClearLayers bool `json:"clear_layers"` } + +type ContainerCacheImageInput struct { + DiskId string `json:"disk_id"` + Image *CacheImageInput `json:"image"` +} + +type ContainerCacheImagesInput struct { + Images []*ContainerCacheImageInput `json:"images"` +} diff --git a/pkg/apis/container.go b/pkg/apis/container.go index 1eb93ca6993..55dcb4c513d 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -228,11 +228,44 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error { return nil } +type ContainerVolumeMountDiskPostImageOverlay struct { + Id string `json:"id"` + PathMap map[string]string `json:"path_map"` +} + +type ContainerVolumeMountDiskPostOverlayType string + +const ( + CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH ContainerVolumeMountDiskPostOverlayType = "host_path" + CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE ContainerVolumeMountDiskPostOverlayType = "image" +) + type ContainerVolumeMountDiskPostOverlay struct { // 宿主机底层目录 HostLowerDir []string `json:"host_lower_dir"` // 合并后要挂载到容器的目录 - ContainerTargetDir string `json:"container_target_dir"` + ContainerTargetDir string `json:"container_target_dir"` + Image *ContainerVolumeMountDiskPostImageOverlay `json:"image"` +} + +func (o ContainerVolumeMountDiskPostOverlay) IsEqual(input ContainerVolumeMountDiskPostOverlay) bool { + if o.GetType() != input.GetType() { + return false + } + switch o.GetType() { + case CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH: + return o.ContainerTargetDir == input.ContainerTargetDir + case CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE: + return o.Image.Id == input.Image.Id + } + return false +} + +func (o ContainerVolumeMountDiskPostOverlay) GetType() ContainerVolumeMountDiskPostOverlayType { + if o.Image != nil { + return CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE + } + return CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH } type ContainerVolumeMountDisk struct { diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index 6c530336205..57f72c24cc3 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -124,6 +124,7 @@ type ContainerSaveVolumeMountToImageInput struct { VolumeMountIndex int `json:"volume_mount_index"` VolumeMount *ContainerVolumeMount `json:"volume_mount"` + VolumeMountDirs []string `json:"volume_mount_dirs"` } type ContainerCommitInput struct { diff --git a/pkg/apis/image/consts.go b/pkg/apis/image/consts.go index 925f4044c32..a6bcb513f7c 100644 --- a/pkg/apis/image/consts.go +++ b/pkg/apis/image/consts.go @@ -54,18 +54,20 @@ const ( IMAGE_STORAGE_DRIVER_S3 = "s3" // image properties - IMAGE_OS_ARCH = "os_arch" - IMAGE_OS_DISTRO = "os_distribution" - IMAGE_OS_TYPE = "os_type" - IMAGE_OS_VERSION = "os_version" - IMAGE_DISK_FORMAT = "disk_format" - IMAGE_UEFI_SUPPORT = "uefi_support" - IMAGE_IS_LVM_PARTITION = "is_lvm_partition" - IMAGE_IS_READONLY = "is_readonly" - IMAGE_PARTITION_TYPE = "partition_type" - IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init" - IMAGE_DISABLE_USB_KBD = "disable_usb_kbd" - IMAGE_VDI_PROTOCOL = "vdi_protocol" + IMAGE_OS_ARCH = "os_arch" + IMAGE_OS_DISTRO = "os_distribution" + IMAGE_OS_TYPE = "os_type" + IMAGE_OS_VERSION = "os_version" + IMAGE_DISK_FORMAT = "disk_format" + IMAGE_UEFI_SUPPORT = "uefi_support" + IMAGE_IS_LVM_PARTITION = "is_lvm_partition" + IMAGE_IS_READONLY = "is_readonly" + IMAGE_PARTITION_TYPE = "partition_type" + IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init" + IMAGE_DISABLE_USB_KBD = "disable_usb_kbd" + IMAGE_VDI_PROTOCOL = "vdi_protocol" + IMAGE_INTERNAL_PATH_MAP = "internal_path_map" + IMAGE_USED_BY_POST_OVERLAY = "used_by_post_overlay" IMAGE_STATUS_UPDATING = "updating" ) diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go index fa745c453f4..c4ca772b364 100644 --- a/pkg/compute/container_drivers/volume_mount/disk.go +++ b/pkg/compute/container_drivers/volume_mount/disk.go @@ -17,7 +17,9 @@ package volume_mount import ( "context" + "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/sets" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -35,16 +37,26 @@ type iDiskOverlay interface { validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, obj *models.SDisk) error } +type iDiskPostOverlay interface { + validateData(ctx context.Context, userCred mcclient.TokenCredential, pov *apis.ContainerVolumeMountDiskPostOverlay) error + getContainerTargetDirs(ov *apis.ContainerVolumeMountDiskPostOverlay) []string +} + type disk struct { - overlayDrivers map[apis.ContainerDiskOverlayType]iDiskOverlay + overlayDrivers map[apis.ContainerDiskOverlayType]iDiskOverlay + postOverlayDrivers map[apis.ContainerVolumeMountDiskPostOverlayType]iDiskPostOverlay } -func newDisk() models.IContainerVolumeMountDriver { +func newDisk() models.IContainerVolumeMountDiskDriver { return &disk{ overlayDrivers: map[apis.ContainerDiskOverlayType]iDiskOverlay{ apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(), apis.CONTAINER_DISK_OVERLAY_TYPE_DISK_IMAGE: newDiskOverlayImage(), }, + postOverlayDrivers: map[apis.ContainerVolumeMountDiskPostOverlayType]iDiskPostOverlay{ + apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH: newDiskPostOverlayHostPath(), + apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE: newDiskPostOverlayImage(), + }, } } @@ -135,7 +147,7 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil { return nil, errors.Wrapf(err, "validate overlay") } - if err := d.ValidatePostOverlay(vm); err != nil { + if err := d.ValidatePostOverlay(ctx, userCred, vm); err != nil { return nil, errors.Wrap(err, "validate post overlay") } return vm, nil @@ -173,6 +185,10 @@ func (d disk) getOverlayDriver(ov *apis.ContainerVolumeMountDiskOverlay) iDiskOv return d.overlayDrivers[ov.GetType()] } +func (d disk) getPostOverlayDriver(pov *apis.ContainerVolumeMountDiskPostOverlay) iDiskPostOverlay { + return d.postOverlayDrivers[pov.GetType()] +} + func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, diskObj *models.SDisk) error { if vm.Disk.Overlay == nil { return nil @@ -187,28 +203,41 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden return nil } -func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error { +func (d disk) ValidatePostSingleOverlay(ctx context.Context, userCred mcclient.TokenCredential, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + drv := d.getPostOverlayDriver(ov) + if err := drv.validateData(ctx, userCred, ov); err != nil { + return errors.Wrapf(err, "validate post overlay %s", ov.GetType()) + } + return nil +} + +func (d disk) ValidatePostOverlayTargetDirs(ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + ctrTargetDirs := sets.NewString() + for _, ov := range ovs { + drv := d.getPostOverlayDriver(ov) + ovCtrTargetDirs := drv.getContainerTargetDirs(ov) + if ctrTargetDirs.HasAny(ovCtrTargetDirs...) { + return httperrors.NewInputParameterError("duplicated container target dirs %v of ov %s", ctrTargetDirs, jsonutils.Marshal(ov)) + } else { + ctrTargetDirs.Insert(ovCtrTargetDirs...) + } + } + return nil +} + +func (d disk) ValidatePostOverlay(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount) error { if len(vm.Disk.PostOverlay) == 0 { return nil } ovs := vm.Disk.PostOverlay - var duplicateCtrDir string - for _, ov := range ovs { - if len(ov.HostLowerDir) == 0 { - return httperrors.NewNotEmptyError("host_lower_dir is required") + for i, ov := range ovs { + if err := d.ValidatePostSingleOverlay(ctx, userCred, ov); err != nil { + return err } - for i, hld := range ov.HostLowerDir { - if len(hld) == 0 { - return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i) - } - } - if len(ov.ContainerTargetDir) == 0 { - return httperrors.NewNotEmptyError("container_target_dir is required") - } - if ov.ContainerTargetDir == duplicateCtrDir { - return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir) - } - duplicateCtrDir = ov.ContainerTargetDir + vm.Disk.PostOverlay[i] = ov + } + if err := d.ValidatePostOverlayTargetDirs(vm.Disk.PostOverlay); err != nil { + return errors.Wrap(err, "validate post overlay target dirs") } if vm.Propagation == "" { // 设置默认 propagation 为 rslave diff --git a/pkg/compute/container_drivers/volume_mount/disk_pov_host_path.go b/pkg/compute/container_drivers/volume_mount/disk_pov_host_path.go new file mode 100644 index 00000000000..c5d67048262 --- /dev/null +++ b/pkg/compute/container_drivers/volume_mount/disk_pov_host_path.go @@ -0,0 +1,35 @@ +package volume_mount + +import ( + "context" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type povHostPath struct { +} + +func newDiskPostOverlayHostPath() iDiskPostOverlay { + return &povHostPath{} +} + +func (p povHostPath) validateData(ctx context.Context, userCred mcclient.TokenCredential, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + if len(ov.HostLowerDir) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir is required") + } + for i, hld := range ov.HostLowerDir { + if len(hld) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i) + } + } + if len(ov.ContainerTargetDir) == 0 { + return httperrors.NewNotEmptyError("container_target_dir is required") + } + return nil +} + +func (p povHostPath) getContainerTargetDirs(ov *apis.ContainerVolumeMountDiskPostOverlay) []string { + return []string{ov.ContainerTargetDir} +} diff --git a/pkg/compute/container_drivers/volume_mount/disk_pov_image.go b/pkg/compute/container_drivers/volume_mount/disk_pov_image.go new file mode 100644 index 00000000000..acca0c8742d --- /dev/null +++ b/pkg/compute/container_drivers/volume_mount/disk_pov_image.go @@ -0,0 +1,70 @@ +package volume_mount + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + imageapi "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + imagemod "yunion.io/x/onecloud/pkg/mcclient/modules/image" +) + +type povImage struct { +} + +func newDiskPostOverlayImage() iDiskPostOverlay { + return &povImage{} +} + +func (p povImage) validateData(ctx context.Context, userCred mcclient.TokenCredential, pov *apis.ContainerVolumeMountDiskPostOverlay) error { + img := pov.Image + if img.Id == "" { + return httperrors.NewMissingParameterError("image id") + } + s := auth.GetAdminSession(ctx, options.Options.Region) + obj, err := imagemod.Images.Get(s, img.Id, nil) + if err != nil { + return errors.Wrapf(err, "Get image by id %s", img.Id) + } + imgObj := new(imageapi.ImageDetails) + if err := obj.Unmarshal(imgObj); err != nil { + return errors.Wrap(err, "unmarshal image details") + } + pov.Image.Id = imgObj.Id + props := imgObj.Properties + usedByStr, ok := props[imageapi.IMAGE_USED_BY_POST_OVERLAY] + if !ok { + return errors.Wrapf(err, "Get %s", imageapi.IMAGE_USED_BY_POST_OVERLAY) + } + if usedByStr != "true" { + return errors.Errorf("image isn't used by post overlay") + } + pathMapStr := props[imageapi.IMAGE_INTERNAL_PATH_MAP] + pathMapObj, err := jsonutils.ParseString(pathMapStr) + if err != nil { + return errors.Wrapf(err, "json parse path_map: %s", pathMapStr) + } + pathMap := make(map[string]string) + if err := pathMapObj.Unmarshal(pathMap); err != nil { + return errors.Wrapf(err, "unmarshal pathMapObj") + } + if len(pov.Image.PathMap) == 0 { + pov.Image.PathMap = pathMap + } + return nil +} + +func (p povImage) getContainerTargetDirs(ov *apis.ContainerVolumeMountDiskPostOverlay) []string { + pathMap := ov.Image.PathMap + ctrPaths := []string{} + for _, ctrPath := range pathMap { + ctrPaths = append(ctrPaths, ctrPath) + } + return ctrPaths +} diff --git a/pkg/compute/models/container_drivers.go b/pkg/compute/models/container_drivers.go index 332a76b81c1..b2015262afc 100644 --- a/pkg/compute/models/container_drivers.go +++ b/pkg/compute/models/container_drivers.go @@ -95,6 +95,14 @@ type IContainerVolumeMountDriver interface { ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) } +type IContainerVolumeMountDiskDriver interface { + IContainerVolumeMountDriver + + ValidatePostOverlay(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount) error + ValidatePostSingleOverlay(ctx context.Context, userCred mcclient.TokenCredential, pov *apis.ContainerVolumeMountDiskPostOverlay) error + ValidatePostOverlayTargetDirs(ovs []*apis.ContainerVolumeMountDiskPostOverlay) error +} + func RegisterContainerDeviceDriver(drv IContainerDeviceDriver) { registerContainerDriver(containerDeviceDrivers, drv.GetType(), drv) } diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index ae78e933a84..0a51f148178 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -755,6 +755,16 @@ func (c *SContainer) PrepareSaveImage(ctx context.Context, userCred mcclient.Tok // inherit the ownership of disk ProjectId: c.ProjectId, } + if len(input.Dirs) > 0 { + dirMap := make(map[string]string, 0) + for _, dir := range input.Dirs { + dirMap[dir] = dir + } + imageInput.Properties[imageapi.IMAGE_INTERNAL_PATH_MAP] = jsonutils.Marshal(dirMap).String() + } + if input.UsedByPostOverlay { + imageInput.Properties[imageapi.IMAGE_USED_BY_POST_OVERLAY] = jsonutils.Marshal(input.UsedByPostOverlay).String() + } // check class metadata cm, err := c.GetAllClassMetadata() if err != nil { @@ -792,6 +802,7 @@ func (c *SContainer) PerformSaveVolumeMountImage(ctx context.Context, userCred m ImageId: imageId, VolumeMountIndex: input.Index, VolumeMount: hvm, + VolumeMountDirs: input.Dirs, } return hostInput, c.StartSaveVolumeMountImage(ctx, userCred, hostInput, "") @@ -1012,16 +1023,17 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre return task.ScheduleRun(nil) } -func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool { - for _, cov := range vm.Disk.PostOverlay { - if ov.ContainerTargetDir == cov.ContainerTargetDir { - return true +func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMountDiskPostOverlay, bool) { + for i := range vm.Disk.PostOverlay { + cov := vm.Disk.PostOverlay[i] + if cov.IsEqual(*ov) { + return cov, true } } - return false + return nil, false } -func (c *SContainer) validateVolumeMountPostOverlayAction(action string, index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) { +func (c *SContainer) validateVolumeMountPostOverlayAction(action string, index int) (*apis.ContainerVolumeMount, error) { if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) { return nil, httperrors.NewInvalidStatusError("can't %s post overlay on status %s", action, c.Status) } @@ -1083,19 +1095,40 @@ func (c *SContainer) GetRemovePostOverlayVolumeMount(index int, ovs []*apis.Cont } func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountAddPostOverlayInput) (jsonutils.JSONObject, error) { - vm, err := c.validateVolumeMountPostOverlayAction("add", input.Index, input.PostOverlay) + vm, err := c.validateVolumeMountPostOverlayAction("add", input.Index) if err != nil { return nil, err } - for _, ov := range input.PostOverlay { - isExist := c.isPostOverlayExist(vm, ov) + totalOvs := []*apis.ContainerVolumeMountDiskPostOverlay{} + totalOvs = append(totalOvs, vm.Disk.PostOverlay...) + drv := GetContainerVolumeMountDriver(vm.Type) + dDrv := drv.(IContainerVolumeMountDiskDriver) + for i := range input.PostOverlay { + ov := input.PostOverlay[i] + cov, isExist := c.isPostOverlayExist(vm, ov) if isExist { - return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir) + return nil, httperrors.NewInputParameterError("post overlay already exists: %s", jsonutils.Marshal(cov)) } + if err := dDrv.ValidatePostSingleOverlay(ctx, userCred, ov); err != nil { + return nil, errors.Wrapf(err, "validate post overlay %s", jsonutils.Marshal(ov)) + } + totalOvs = append(totalOvs, ov) + } + if err := dDrv.ValidatePostOverlayTargetDirs(totalOvs); err != nil { + return nil, errors.Wrapf(err, "validate container target dirs") } return nil, c.StartAddVolumeMountPostOverlayTask(ctx, userCred, input, "") } +func (c *SContainer) StartCacheImagesTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerCacheImagesInput, parentTaskId string) error { + c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_CACHE_IMAGE, "") + task, err := taskman.TaskManager.NewTask(ctx, "ContainerCacheImagesTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "New ContainerCacheImagesTask") + } + return task.ScheduleRun(nil) +} + func (c *SContainer) StartAddVolumeMountPostOverlayTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerVolumeMountAddPostOverlayInput, parentTaskId string) error { c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_ADD_POST_OVERLY, "") task, err := taskman.TaskManager.NewTask(ctx, "ContainerAddVolumeMountPostOverlayTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil) @@ -1115,18 +1148,19 @@ func (c *SContainer) StartRemoveVolumeMountPostOverlayTask(ctx context.Context, } func (c *SContainer) PerformRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountRemovePostOverlayInput) (jsonutils.JSONObject, error) { - vm, err := c.validateVolumeMountPostOverlayAction("remove", input.Index, input.PostOverlay) + vm, err := c.validateVolumeMountPostOverlayAction("remove", input.Index) if err != nil { return nil, err } if len(vm.Disk.PostOverlay) == 0 { return nil, httperrors.NewInputParameterError("no post overlay") } - for _, ov := range input.PostOverlay { - isExist := c.isPostOverlayExist(vm, ov) + for i, ov := range input.PostOverlay { + cov, isExist := c.isPostOverlayExist(vm, ov) if !isExist { - return nil, httperrors.NewInputParameterError("post overlay %s not exists", ov.ContainerTargetDir) + return nil, httperrors.NewInputParameterError("post overlay not exists: %s", jsonutils.Marshal(ov)) } + input.PostOverlay[i] = cov } return nil, c.StartRemoveVolumeMountPostOverlayTask(ctx, userCred, input, "") } @@ -1134,7 +1168,7 @@ func (c *SContainer) PerformRemoveVolumeMountPostOverlay(ctx context.Context, us func (c *SContainer) removePostOverlay(vmd *apis.ContainerVolumeMountDisk, ov *apis.ContainerVolumeMountDiskPostOverlay) *apis.ContainerVolumeMountDisk { curOvs := vmd.PostOverlay for i, cov := range curOvs { - if cov.ContainerTargetDir == ov.ContainerTargetDir { + if cov.IsEqual(*ov) { curOvs = append(curOvs[:i], curOvs[i+1:]...) } } diff --git a/pkg/compute/tasks/container_cache_images_task.go b/pkg/compute/tasks/container_cache_images_task.go new file mode 100644 index 00000000000..0720c829457 --- /dev/null +++ b/pkg/compute/tasks/container_cache_images_task.go @@ -0,0 +1,98 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" +) + +func init() { + taskman.RegisterTask(ContainerCacheImagesTask{}) +} + +type ContainerCacheImagesTask struct { + ContainerBaseTask +} + +func (t *ContainerCacheImagesTask) getInput() (*api.ContainerCacheImagesInput, error) { + input := new(api.ContainerCacheImagesInput) + if err := t.GetParams().Unmarshal(input); err != nil { + return nil, err + } + return input, nil +} + +func (t *ContainerCacheImagesTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + if err := t.startCacheImages(ctx, obj); err != nil { + t.onError(ctx, obj.(*models.SContainer), jsonutils.NewString(err.Error())) + return + } +} + +func (t *ContainerCacheImagesTask) onError(ctx context.Context, ctr *models.SContainer, reason jsonutils.JSONObject) { + ctr.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_CACHE_IMAGE_FAILED, reason.String()) + t.SetStageFailed(ctx, reason) +} + +func (t *ContainerCacheImagesTask) startCacheImages(ctx context.Context, obj db.IStandaloneModel) error { + input, err := t.getInput() + if err != nil { + return errors.Wrapf(err, "getInput") + } + //caches := make([]db.IStandaloneModel, 0) + //params := []*api.CacheImageInput{} + t.SetStage("OnStorageCacheImageComplete", nil) + for i, img := range input.Images { + disk := models.DiskManager.FetchDiskById(img.DiskId) + if disk == nil { + return errors.Wrapf(httperrors.ErrNotFound, "disk not found %s", img.DiskId) + } + storage, _ := disk.GetStorage() + storagecache := storage.GetStoragecache() + if storagecache == nil { + return errors.Wrapf(httperrors.ErrNotFound, "storage cache not found by %s", storage.GetId()) + } + //caches = append(caches, storagecache) + param := input.Images[i].Image + param.ParentTaskId = t.GetTaskId() + //params = append(params, param) + if err := storagecache.StartImageCacheTask(ctx, t.GetUserCred(), *param); err != nil { + return errors.Wrapf(err, "startImageCacheTask of param: %s", jsonutils.Marshal(param)) + } + } + return nil +} + +func (t *ContainerCacheImagesTask) OnStorageCacheImageComplete(ctx context.Context, ctr *models.SContainer, data jsonutils.JSONObject) { + if t.IsSubtask() { + t.SetStageComplete(ctx, nil) + return + } + ctr.StartSyncStatusTask(ctx, t.GetUserCred(), t.GetTaskId()) + t.SetStageComplete(ctx, nil) +} + +func (t *ContainerCacheImagesTask) OnStorageCacheImageCompleteFailed(ctx context.Context, ctr *models.SContainer, data jsonutils.JSONObject) { + t.onError(ctx, ctr, data) +} diff --git a/pkg/compute/tasks/container_volume_mount_task.go b/pkg/compute/tasks/container_volume_mount_task.go index 3ce13852603..085704ce287 100644 --- a/pkg/compute/tasks/container_volume_mount_task.go +++ b/pkg/compute/tasks/container_volume_mount_task.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" + imageapi "yunion.io/x/onecloud/pkg/apis/image" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -51,7 +52,54 @@ type ContainerAddVolumeMountPostOverlayTask struct { } func (t *ContainerAddVolumeMountPostOverlayTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { - t.requestAdd(ctx, obj.(*models.SContainer)) + ctr := obj.(*models.SContainer) + if err := t.startCacheImage(ctx, ctr); err != nil { + t.OnAddedFailed(ctx, ctr, jsonutils.NewString(err.Error())) + } +} + +func (t *ContainerAddVolumeMountPostOverlayTask) startCacheImage(ctx context.Context, ctr *models.SContainer) error { + input, err := t.getInput() + if err != nil { + return errors.Wrap(err, "getInput") + } + volIdx := input.Index + vol := ctr.Spec.VolumeMounts[volIdx] + if vol.Disk == nil || vol.Disk.Id == "" { + return errors.Wrapf(err, "invalid volume mount disk %s", jsonutils.Marshal(vol.Disk).String()) + } + diskId := vol.Disk.Id + taskInput := &api.ContainerCacheImagesInput{ + Images: make([]*api.ContainerCacheImageInput, 0), + } + for i := range input.PostOverlay { + po := input.PostOverlay[i] + if po.GetType() == apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE { + taskInput.Images = append(taskInput.Images, &api.ContainerCacheImageInput{ + DiskId: diskId, + Image: &api.CacheImageInput{ + ImageId: po.Image.Id, + Format: imageapi.IMAGE_DISK_FORMAT_TGZ, + SkipChecksumIfExists: true, + }, + }) + } + } + if len(taskInput.Images) != 0 { + t.SetStage("OnCachedImagesComplete", nil) + return ctr.StartCacheImagesTask(ctx, t.GetUserCred(), taskInput, t.GetTaskId()) + } else { + t.OnCachedImagesComplete(ctx, ctr, nil) + return nil + } +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnCachedImagesComplete(ctx context.Context, ctr *models.SContainer, data jsonutils.JSONObject) { + t.requestAdd(ctx, ctr) +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnCachedImagesCompleteFailed(ctx context.Context, ctr *models.SContainer, data jsonutils.JSONObject) { + t.OnAddedFailed(ctx, ctr, jsonutils.NewString(data.String())) } func (t *ContainerAddVolumeMountPostOverlayTask) getInput() (*api.ContainerVolumeMountAddPostOverlayInput, error) { diff --git a/pkg/compute/tasks/storage_batch_cache_image_task.go b/pkg/compute/tasks/storage_batch_cache_image_task.go new file mode 100644 index 00000000000..0a816ed8770 --- /dev/null +++ b/pkg/compute/tasks/storage_batch_cache_image_task.go @@ -0,0 +1,54 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(StorageBatchCacheImageTask{}) +} + +type StorageBatchCacheImageTask struct { + taskman.STask +} + +func (t *StorageBatchCacheImageTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, data jsonutils.JSONObject) { + t.SetStage("OnStorageCacheImageComplete", nil) + params := make([]api.CacheImageInput, 0) + t.GetParams().Unmarshal(¶ms, "params") + for i := range objs { + sc := objs[i].(*models.SStoragecache) + input := params[i] + input.ParentTaskId = t.GetTaskId() + if err := sc.StartImageCacheTask(ctx, t.GetUserCred(), input); err != nil { + t.SetStageFailed(ctx, jsonutils.NewString(fmt.Sprintf("start image cache task %s failed: %s", sc.GetId(), err))) + return + } + } +} + +func (t *StorageBatchCacheImageTask) OnStorageCacheImageComplete(ctx context.Context, obj []db.IStandaloneModel, data jsonutils.JSONObject) { + t.SetStageComplete(ctx, nil) +} diff --git a/pkg/hostman/container/volume_mount/disk/disk.go b/pkg/hostman/container/volume_mount/disk/disk.go index 3ddf9264672..4d49caf898c 100644 --- a/pkg/hostman/container/volume_mount/disk/disk.go +++ b/pkg/hostman/container/volume_mount/disk/disk.go @@ -19,11 +19,13 @@ import ( "fmt" "path/filepath" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" + imageapi "yunion.io/x/onecloud/pkg/apis/image" "yunion.io/x/onecloud/pkg/hostman/container/storage" container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" @@ -277,25 +279,38 @@ func (d disk) mountOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi. return d.getOverlayDriver(vm.Disk.Overlay).mount(d, pod, ctrId, vm) } -func (d disk) doTemplateOverlayAction( - ctx context.Context, - pod volume_mount.IPodInfo, ctrId string, - vm *hostapi.ContainerVolumeMount, - ovAction func(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { - templateId := vm.Disk.TemplateId +func (d disk) getCachedImageDir(ctx context.Context, pod volume_mount.IPodInfo, imgId string) (string, error) { input := computeapi.CacheImageInput{ - ImageId: templateId, + ImageId: imgId, + Format: imageapi.IMAGE_DISK_FORMAT_TGZ, SkipChecksumIfExists: true, } cachedImgMan := storageman.GetManager().LocalStorageImagecacheManager + logPrefx := fmt.Sprintf("pod %s, image %s", pod.GetName(), imgId) + log.Infof("%s try to accuire image...", logPrefx) cachedImg, err := cachedImgMan.AcquireImage(ctx, input, nil) if err != nil { - return errors.Wrapf(err, "Get cache image %s", templateId) + return "", errors.Wrapf(err, "Get cache image %s", imgId) } - defer cachedImgMan.ReleaseImage(ctx, templateId) + defer cachedImgMan.ReleaseImage(ctx, imgId) + log.Infof("%s try to get access directory", logPrefx) cachedImageDir, err := cachedImg.GetAccessDirectory() if err != nil { - return errors.Wrapf(err, "GetAccessDirectory of cached image %s", cachedImg.GetPath()) + return "", errors.Wrapf(err, "GetAccessDirectory of cached image %s", cachedImg.GetPath()) + } + log.Infof("%s got cached image dir %s", logPrefx, cachedImageDir) + return cachedImageDir, nil +} + +func (d disk) doTemplateOverlayAction( + ctx context.Context, + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ovAction func(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { + templateId := vm.Disk.TemplateId + cachedImageDir, err := d.getCachedImageDir(ctx, pod, templateId) + if err != nil { + return errors.Wrap(err, "get cached image dir") } vm.Disk.Overlay = &apis.ContainerVolumeMountDiskOverlay{ LowerDir: []string{cachedImageDir}, diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay.go b/pkg/hostman/container/volume_mount/disk/post_overlay.go index 24f7ea54418..fb552018b04 100644 --- a/pkg/hostman/container/volume_mount/disk/post_overlay.go +++ b/pkg/hostman/container/volume_mount/disk/post_overlay.go @@ -23,7 +23,6 @@ import ( "yunion.io/x/onecloud/pkg/apis" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" - "yunion.io/x/onecloud/pkg/util/mountutils" ) const ( @@ -74,9 +73,31 @@ func newDiskPostOverlay(d disk) iDiskPostOverlay { } } +var ( + postOverlayDrivers = make(map[apis.ContainerVolumeMountDiskPostOverlayType]iDiskPostOverlayDriver) +) + +func registerPostOverlayDriver(drv iDiskPostOverlayDriver) { + postOverlayDrivers[drv.GetType()] = drv +} + +func getPostOverlayDriver(typ apis.ContainerVolumeMountDiskPostOverlayType) iDiskPostOverlayDriver { + return postOverlayDrivers[typ] +} + +type iDiskPostOverlayDriver interface { + GetType() apis.ContainerVolumeMountDiskPostOverlayType + Mount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error + Unmount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, clearLayers bool) error +} + +func (d diskPostOverlay) getDriver(ov *apis.ContainerVolumeMountDiskPostOverlay) iDiskPostOverlayDriver { + return getPostOverlayDriver(ov.GetType()) +} + func (d diskPostOverlay) mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { for _, ov := range ovs { - if err := d.mountPostOverlay(pod, ctrId, vm, ov); err != nil { + if err := d.getDriver(ov).Mount(d, pod, ctrId, vm, ov); err != nil { return errors.Wrapf(err, "mount container %s post overlay dir: %#v", ctrId, ov) } } @@ -85,7 +106,7 @@ func (d diskPostOverlay) mountPostOverlays(pod volume_mount.IPodInfo, ctrId stri func (d diskPostOverlay) unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, clearLayers bool) error { for _, ov := range ovs { - if err := d.unmountPostOverlay(pod, ctrId, vm, ov, useLazy, clearLayers); err != nil { + if err := d.getDriver(ov).Unmount(d, pod, ctrId, vm, ov, useLazy, clearLayers); err != nil { return errors.Wrapf(err, "unmount container %s post overlay dir: %#v", ctrId, ov) } } @@ -144,52 +165,3 @@ func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctr } return mergedDir, nil } - -func (d diskPostOverlay) mountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { - upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) - if err != nil { - return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) - } - - workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, true) - if err != nil { - return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) - } - - mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) - if err != nil { - return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) - } - - return mountutils.MountOverlayWithFeatures(ov.HostLowerDir, upperDir, workDir, mergedDir, &mountutils.MountOverlayFeatures{ - MetaCopy: true, - }) -} - -func (d diskPostOverlay) unmountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { - mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) - if err != nil { - return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) - } - if err := mountutils.Unmount(mergedDir, useLazy); err != nil { - return errors.Wrapf(err, "unmount %s", mergedDir) - } - if cleanLayers { - upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, false) - if err != nil { - return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) - } - if err := volume_mount.RemoveDir(upperDir); err != nil { - return errors.Wrap(err, "remove upper dir") - } - - workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, false) - if err != nil { - return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) - } - if err := volume_mount.RemoveDir(workDir); err != nil { - return errors.Wrap(err, "remove work dir") - } - } - return nil -} diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go b/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go new file mode 100644 index 00000000000..3051c977d34 --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go @@ -0,0 +1,88 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/util/mountutils" +) + +func init() { + registerPostOverlayDriver(newPostOverlayHostPath()) +} + +func newPostOverlayHostPath() iDiskPostOverlayDriver { + return &postOverlayHostPath{} +} + +type postOverlayHostPath struct { +} + +func (p postOverlayHostPath) GetType() apis.ContainerVolumeMountDiskPostOverlayType { + return apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH +} + +func (p postOverlayHostPath) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + + return mountutils.MountOverlayWithFeatures(ov.HostLowerDir, upperDir, workDir, mergedDir, &mountutils.MountOverlayFeatures{ + MetaCopy: true, + }) +} + +func (p postOverlayHostPath) Unmount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + if err := mountutils.Unmount(mergedDir, useLazy); err != nil { + return errors.Wrapf(err, "unmount %s", mergedDir) + } + if cleanLayers { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + if err := volume_mount.RemoveDir(upperDir); err != nil { + return errors.Wrap(err, "remove upper dir") + } + + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + if err := volume_mount.RemoveDir(workDir); err != nil { + return errors.Wrap(err, "remove work dir") + } + } + return nil +} diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay_image.go b/pkg/hostman/container/volume_mount/disk/post_overlay_image.go new file mode 100644 index 00000000000..7ed1a6a51f2 --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/post_overlay_image.go @@ -0,0 +1,97 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "context" + "path/filepath" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" +) + +func init() { + registerPostOverlayDriver(newPostOverlayImage()) +} + +func newPostOverlayImage() iDiskPostOverlayDriver { + return &postOverlayImage{} +} + +type postOverlayImage struct { +} + +func (i postOverlayImage) GetType() apis.ContainerVolumeMountDiskPostOverlayType { + return apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_IMAGE +} + +func (i postOverlayImage) getImageInput(ov *apis.ContainerVolumeMountDiskPostOverlay) *apis.ContainerVolumeMountDiskPostImageOverlay { + return ov.Image +} + +func (i postOverlayImage) getCachedImagePaths(d diskPostOverlay, pod volume_mount.IPodInfo, img *apis.ContainerVolumeMountDiskPostImageOverlay) (map[string]string, error) { + cachedImgDir, err := d.disk.getCachedImageDir(context.Background(), pod, img.Id) + if err != nil { + return nil, errors.Wrap(err, "disk.getCachedImageDir") + } + result := make(map[string]string) + for hostPathSuffix, ctrPath := range img.PathMap { + hostPath := filepath.Join(cachedImgDir, hostPathSuffix) + result[hostPath] = ctrPath + } + return result, nil +} + +func (i postOverlayImage) convertToDiskOV(ov *apis.ContainerVolumeMountDiskPostOverlay, hostPath, ctrPath string) *apis.ContainerVolumeMountDiskPostOverlay { + return &apis.ContainerVolumeMountDiskPostOverlay{ + HostLowerDir: []string{hostPath}, + ContainerTargetDir: ctrPath, + } +} + +func (i postOverlayImage) withAction( + d diskPostOverlay, pod volume_mount.IPodInfo, ov *apis.ContainerVolumeMountDiskPostOverlay, + af func(iDiskPostOverlayDriver, *apis.ContainerVolumeMountDiskPostOverlay) error) error { + img := i.getImageInput(ov) + paths, err := i.getCachedImagePaths(d, pod, img) + if err != nil { + return errors.Wrapf(err, "get cached image paths") + } + for hostPath, ctrPath := range paths { + dov := i.convertToDiskOV(ov, hostPath, ctrPath) + drv := d.getDriver(dov) + if err := af(drv, dov); err != nil { + return errors.Wrapf(err, "host path %s to %s", hostPath, ctrPath) + } + } + return nil +} + +func (i postOverlayImage) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + return i.withAction(d, pod, ov, + func(driver iDiskPostOverlayDriver, dov *apis.ContainerVolumeMountDiskPostOverlay) error { + return driver.Mount(d, pod, ctrId, vm, dov) + }) +} + +func (i postOverlayImage) Unmount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, clearLayers bool) error { + return i.withAction(d, pod, ov, + func(driver iDiskPostOverlayDriver, dov *apis.ContainerVolumeMountDiskPostOverlay) error { + return driver.Unmount(d, pod, ctrId, vm, dov, useLazy, clearLayers) + }) +} diff --git a/pkg/hostman/container/volume_mount/interface.go b/pkg/hostman/container/volume_mount/interface.go index f3c90bbdf88..ec0ec8d0669 100644 --- a/pkg/hostman/container/volume_mount/interface.go +++ b/pkg/hostman/container/volume_mount/interface.go @@ -43,6 +43,7 @@ func GetDriver(typ apis.ContainerVolumeMountType) IVolumeMount { } type IPodInfo interface { + GetName() string GetVolumesDir() string GetVolumesOverlayDir() string GetDisks() []*desc.SGuestDisk diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 51c9d1ca631..43f5556a833 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -2071,7 +2071,11 @@ func (s *sPodGuestInstance) SaveVolumeMountToImage(ctx context.Context, userCred func (s *sPodGuestInstance) tarGzDir(input *hostapi.ContainerSaveVolumeMountToImageInput, ctrId string, hostPath string) (string, error) { fp := fmt.Sprintf("volimg-%s-ctr-%s-%d.tar.gz", input.ImageId, ctrId, input.VolumeMountIndex) outputFp := filepath.Join(s.GetVolumesDir(), fp) - cmd := fmt.Sprintf("tar -czf %s -C %s .", outputFp, hostPath) + dirPath := "." + if len(input.VolumeMountDirs) != 0 { + dirPath = strings.Join(input.VolumeMountDirs, " ") + } + cmd := fmt.Sprintf("tar -czf %s -C %s %s", outputFp, hostPath, dirPath) if out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output(); err != nil { return "", errors.Wrapf(err, "%s: %s", cmd, out) } diff --git a/pkg/hostman/storageman/imagecache_local.go b/pkg/hostman/storageman/imagecache_local.go index 1577e732339..78773ed2fc2 100644 --- a/pkg/hostman/storageman/imagecache_local.go +++ b/pkg/hostman/storageman/imagecache_local.go @@ -340,14 +340,19 @@ func (l *SLocalImageCache) GetAccessDirectory() (string, error) { if fileutils2.Exists(dir) { return dir, nil } + tmpDir := fmt.Sprintf("%s-tmp", dir) // untar cached image - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", tmpDir).Output() if err != nil { - return "", errors.Wrapf(err, "mkdir %s: %s", dir, out) + return "", errors.Wrapf(err, "mkdir %s: %s", tmpDir, out) } - out, err = procutils.NewRemoteCommandAsFarAsPossible("tar", "xf", l.GetPath(), "-C", dir).Output() + out, err = procutils.NewRemoteCommandAsFarAsPossible("tar", "xf", l.GetPath(), "-C", tmpDir).Output() if err != nil { - return "", errors.Wrapf(err, "untar to %s: %s", dir, out) + return "", errors.Wrapf(err, "untar to %s: %s", tmpDir, out) + } + out, err = procutils.NewRemoteCommandAsFarAsPossible("mv", tmpDir, dir).Output() + if err != nil { + return "", errors.Wrapf(err, "mv %s %s: %s", tmpDir, dir, out) } return dir, nil diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index 47fa2aa4900..b7401df7edd 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -302,18 +302,22 @@ type ContainerStartOptions struct { type ContainerSaveVolumeMountImage struct { options.ResourceIdOptions - IMAGENAME string `help:"Image name"` - INDEX int `help:"Index of volume mount"` - GenerateName string `help:"Generate image name automatically"` - Notes string `help:"Extra notes of the image"` + IMAGENAME string `help:"Image name"` + INDEX int `help:"Index of volume mount"` + GenerateName string `help:"Generate image name automatically"` + Notes string `help:"Extra notes of the image"` + UsedByPostOverlay bool `help:"Used by voluem mount post-overlay"` + Dirs []string `help:"Internal directories"` } func (o ContainerSaveVolumeMountImage) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(&computeapi.ContainerSaveVolumeMountToImageInput{ - Name: o.IMAGENAME, - GenerateName: o.GenerateName, - Notes: o.Notes, - Index: o.INDEX, + Name: o.IMAGENAME, + GenerateName: o.GenerateName, + Notes: o.Notes, + Index: o.INDEX, + Dirs: o.Dirs, + UsedByPostOverlay: o.UsedByPostOverlay, }), nil } @@ -456,6 +460,7 @@ type ContainerAddVolumeMountPostOverlayOptions struct { ServerIdOptions INDEX int `help:"INDEX of volume mount"` MountDesc []string `help:"Mount description, :" short-token:"m"` + Image []string `help:"Image name or id"` } func (o *ContainerAddVolumeMountPostOverlayOptions) Params() (jsonutils.JSONObject, error) { @@ -475,6 +480,15 @@ func (o *ContainerAddVolumeMountPostOverlayOptions) Params() (jsonutils.JSONObje ContainerTargetDir: containerTargetDir, }) } + if len(o.Image) != 0 { + for _, img := range o.Image { + input.PostOverlay = append(input.PostOverlay, &apis.ContainerVolumeMountDiskPostOverlay{ + Image: &apis.ContainerVolumeMountDiskPostImageOverlay{ + Id: img, + }, + }) + } + } return jsonutils.Marshal(input), nil }