fix: xfs zerofree (experimental) (#18501)

Co-authored-by: Qiu Jian <[email protected]>

pkg/hostman/diskutils/fsutils/fsutils.go

@@ -28,10 +28,10 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"
 
-	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/regutils2"
+	"yunion.io/x/onecloud/pkg/util/xfsutils"
 )
 
 func IsPartedFsString(fsstr string) bool {
@@ -261,8 +261,8 @@ func ResizePartitionFs(fpath, fs string, raiseError bool) (error, bool) {
 		FsckXfsFs(fpath)
 		uuid := uuids["UUID"]
 		if len(uuid) > 0 {
-			kvmpart.LockXfsPartition(uuid)
-			defer kvmpart.UnlockXfsPartition(uuid)
+			xfsutils.LockXfsPartition(uuid)
+			defer xfsutils.UnlockXfsPartition(uuid)
 		}
 		cmds = [][]string{{"mkdir", "-p", tmpPoint},
 			{"mount", fpath, tmpPoint},

pkg/hostman/diskutils/nbd/nbdman.go

@@ -17,10 +17,14 @@ package nbd
 import (
 	"fmt"
 	"sync"
+	"time"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
 type SNBDManager struct {
@@ -44,26 +48,89 @@ func NewNBDManager() (*SNBDManager, error) {
 	var ret = new(SNBDManager)
 	ret.nbdDevs = make(map[string]bool, 0)
 	ret.nbdLock = new(sync.Mutex)
+
+	ret.cleanupNbdDevices()
+	if err := ret.reloadNbdDevices(); err != nil {
+		return ret, errors.Wrap(err, "reloadNbdDevices")
+	}
 	if err := ret.findNbdDevices(); err != nil {
-		return ret, err
+		return ret, errors.Wrap(err, "findNbdDevices")
 	}
 	return ret, nil
 }
 
+func tryDetachNbd(nbddev string) error {
+	const MaxTries = 3
+	tried := 0
+	var errs []error
+	for tried < MaxTries && fileutils2.IsBlockDeviceUsed(nbddev) {
+		tried++
+		err := QemuNbdDisconnect(nbddev)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		time.Sleep(time.Second)
+	}
+	if tried < MaxTries {
+		return nil
+	}
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	return nil
+}
+
+func (m *SNBDManager) cleanupNbdDevices() {
+	var i = 0
+	for {
+		nbddev := fmt.Sprintf("/dev/nbd%d", i)
+		if fileutils2.Exists(nbddev) {
+			if fileutils2.IsBlockDeviceUsed(nbddev) {
+				err := tryDetachNbd(nbddev)
+				if err != nil {
+					log.Errorf("tryDetachNbd fail %s", err)
+				}
+			}
+		} else {
+			break
+		}
+	}
+}
+
+func (m *SNBDManager) reloadNbdDevices() error {
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("rmmod", "nbd").Output()
+	if err != nil {
+		log.Errorf("rmmod error: %s", output)
+	}
+	output, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "nbd", "max_part=16").Output()
+	if err != nil {
+		return errors.Wrapf(err, "Failed to activate nbd device: %s", output)
+	}
+	return nil
+}
+
 func (m *SNBDManager) findNbdDevices() error {
 	var i = 0
 	for {
-		if fileutils2.Exists(fmt.Sprintf("/dev/nbd%d", i)) {
-			m.nbdDevs[fmt.Sprintf("/dev/nbd%d", i)] = false
+		nbddev := fmt.Sprintf("/dev/nbd%d", i)
+		if fileutils2.Exists(nbddev) {
+			if fileutils2.IsBlockDeviceUsed(nbddev) {
+				continue
+			}
+			m.nbdDevs[nbddev] = false
+			// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-bdi
+			nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i)
+			sysutils.SetSysConfig(nbdBdi+"max_ratio", "0")
+			sysutils.SetSysConfig(nbdBdi+"min_ratio", "0")
 			i++
 		} else {
 			break
 		}
 	}
-	log.Infof("NBD_DEVS: %#v", m.nbdDevs)
 	if len(m.nbdDevs) == 0 {
-		return fmt.Errorf("No nbd devices found")
+		return errors.Wrap(errors.ErrNotFound, "No nbd devices found")
 	}
+	log.Infof("NBD_DEVS: %#v", m.nbdDevs)
 	return nil
 }
 
@@ -73,6 +140,7 @@ func (m *SNBDManager) AcquireNbddev() string {
 	for nbdDev := range m.nbdDevs {
 		if fileutils2.IsBlockDeviceUsed(nbdDev) {
 			m.nbdDevs[nbdDev] = true
+			continue
 		}
 		if !m.nbdDevs[nbdDev] {
 			m.nbdDevs[nbdDev] = true
@@ -86,6 +154,8 @@ func (m *SNBDManager) ReleaseNbddev(nbddev string) {
 	if _, ok := m.nbdDevs[nbddev]; ok {
 		defer m.nbdLock.Unlock()
 		m.nbdLock.Lock()
-		m.nbdDevs[nbddev] = false
+		if !fileutils2.IsBlockDeviceUsed(nbddev) {
+			m.nbdDevs[nbddev] = false
+		}
 	}
 }

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -19,18 +19,22 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/util/btrfsutils"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/xfsutils"
 )
 
 type SKVMGuestDiskPartition struct {
@@ -173,12 +177,12 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 	var err error
 	if fsType == "xfs" {
 		uuids, _ := fileutils2.GetDevUuid(p.partDev)
-		p.uuid, _ = uuids["UUID"]
+		p.uuid = uuids["UUID"]
 		if len(p.uuid) > 0 {
-			LockXfsPartition(p.uuid)
+			xfsutils.LockXfsPartition(p.uuid)
 			defer func() {
 				if err != nil {
-					UnlockXfsPartition(p.uuid)
+					xfsutils.UnlockXfsPartition(p.uuid)
 				}
 			}()
 		}
@@ -263,7 +267,7 @@ func (p *SKVMGuestDiskPartition) Umount() error {
 
 	defer func() {
 		if p.fs == "xfs" && len(p.uuid) > 0 {
-			UnlockXfsPartition(p.uuid)
+			xfsutils.UnlockXfsPartition(p.uuid)
 		}
 	}()
 
@@ -276,12 +280,14 @@ func (p *SKVMGuestDiskPartition) Umount() error {
 	}
 
 	// check lsof
-	{
+	for {
 		out, err := procutils.NewCommand("lsof", p.mountPath).Output()
 		if err != nil {
 			log.Warningf("lsof %s fail %s", p.mountPath, err)
+			break
 		} else {
 			log.Infof("unmount path %s lsof %s", p.mountPath, string(out))
+			time.Sleep(time.Second * 1)
 		}
 	}
 
@@ -303,7 +309,7 @@ func (p *SKVMGuestDiskPartition) Umount() error {
 			return nil
 		} else {
 			log.Warningf("failed umount %s: %s %s", p.partDev, err, out)
-			time.Sleep(time.Second * 1)
+			time.Sleep(time.Second * 3)
 		}
 	}
 	return errors.Wrapf(err, "umount %s", p.mountPath)
@@ -318,6 +324,8 @@ func (p *SKVMGuestDiskPartition) Zerofree() {
 			p.zerofreeExt()
 		case "ntfs":
 			p.zerofreeNtfs()
+			// case "xfs":
+			//	p.zerofreeFs("xfs")
 		}
 	}
 }
@@ -355,3 +363,82 @@ func (p *SKVMGuestDiskPartition) zerofreeNtfs() {
 		return
 	}
 }
+
+func (p *SKVMGuestDiskPartition) zerofreeFs(fs string) {
+	err := p.zerofreeFsInternal(fs)
+	if err != nil {
+		log.Errorf("zerofreeFs %s %s: %s", p.partDev, fs, err)
+	}
+}
+
+func (p *SKVMGuestDiskPartition) zerofreeFsInternal(fs string) error {
+	uuids, _ := fileutils2.GetDevUuid(p.partDev)
+
+	backupDirPath, err := os.MkdirTemp(consts.DeployTempDir(), "backup")
+	if err != nil {
+		return errors.Wrap(err, "MkdirTemp")
+	}
+	defer func() {
+		cmd := []string{"rm", "-fr", backupDirPath}
+		output, err := procutils.NewCommand(cmd[0], cmd[1:]...).Output()
+		if err != nil {
+			log.Errorf("fail to remove %s %s: %s", backupDirPath, output, err)
+		}
+	}()
+	backupPath := filepath.Join(backupDirPath, "backup.tar.gz")
+
+	err = func() error {
+		// mount partition
+		if !p.Mount() {
+			return errors.Wrap(errors.ErrInvalidStatus, "fail to mount")
+		}
+		defer p.Umount()
+
+		// backup
+		cmd := []string{"tar", "-cpzf", backupPath, "--one-file-system", "-C", p.mountPath, "."}
+		output, err := procutils.NewCommand(cmd[0], cmd[1:]...).Output()
+		if err != nil {
+			return errors.Wrapf(err, "exec %s output %s", cmd, output)
+		}
+
+		return nil
+	}()
+	if err != nil {
+		return errors.Wrap(err, "backup")
+	}
+
+	// zero partition
+	output, err := procutils.NewCommand("shred", "-n", "0", "-z", p.partDev).Output()
+	if err != nil {
+		return errors.Wrapf(err, "shred output %s", output)
+	}
+	// make partition
+	uuid := uuids["UUID"]
+	err = fsutils.FormatPartition(p.partDev, fs, uuid)
+	if err != nil {
+		return errors.Wrapf(err, "FormatPartition %s %s %s", p.partDev, fs, uuid)
+	}
+
+	err = func() error {
+		// mount partition
+		if !p.Mount() {
+			return errors.Wrap(errors.ErrInvalidStatus, "fail to mount")
+		}
+		// umount
+		defer p.Umount()
+
+		// copy back data
+		cmd := []string{"tar", "-xpzf", backupPath, "--numeric-owner", "-C", p.mountPath}
+		output, err := procutils.NewCommand(cmd[0], cmd[1:]...).Output()
+		if err != nil {
+			return errors.Wrapf(err, "exec %s output %s", cmd, output)
+		}
+
+		return nil
+	}()
+	if err != nil {
+		return errors.Wrap(err, "restore")
+	}
+
+	return nil
+}

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -41,7 +41,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
-	"yunion.io/x/onecloud/pkg/util/sysutils"
 	"yunion.io/x/onecloud/pkg/util/winutils"
 )
 
@@ -102,6 +101,7 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 	if err != nil {
 		if errors.Cause(err) == errors.ErrNotFound && req.DeployInfo.IsInit {
 			// if init deploy, ignore no partition error
+			log.Errorf("disk.MountRootfs not found partition, not init, quit")
 			return new(deployapi.DeployGuestFsResponse), nil
 		}
 		log.Errorf("Failed mounting rootfs for %s disk: %s", req.GuestDesc.Hypervisor, err)
@@ -110,9 +110,11 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 	defer disk.UmountRootfs(root)
 	ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo)
 	if err != nil {
+		log.Errorf("guestfs.DoDeployGuestFs fail %s", err)
 		return new(deployapi.DeployGuestFsResponse), err
 	}
 	if ret == nil {
+		log.Errorf("guestfs.DoDeployGuestFs return empty results")
 		return new(deployapi.DeployGuestFsResponse), nil
 	}
 	return ret, nil
@@ -400,29 +402,13 @@ func (s *SDeployService) PrepareEnv() error {
 	if err := s.FixPathEnv(); err != nil {
 		return err
 	}
-	output, err := procutils.NewRemoteCommandAsFarAsPossible("rmmod", "nbd").Output()
-	if err != nil {
-		log.Errorf("rmmod error: %s", output)
-	}
-	output, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "nbd", "max_part=16").Output()
-	if err != nil {
-		return fmt.Errorf("Failed to activate nbd device: %s", output)
-	}
-	err = nbd.Init()
-	if err != nil {
-		return err
-	}
 
-	// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-bdi
-	for i := 0; i < 16; i++ {
-		nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i)
-		sysutils.SetSysConfig(nbdBdi+"max_ratio", "0")
-		sysutils.SetSysConfig(nbdBdi+"min_ratio", "0")
+	if err := nbd.Init(); err != nil {
+		return errors.Wrap(err, "nbd.Init")
 	}
 
 	// create /dev/lvm_remote
-	err = s.checkLvmRemote()
-	if err != nil {
+	if err := s.checkLvmRemote(); err != nil {
 		return errors.Wrap(err, "unable to checkLvmRemote")
 	}
 

pkg/util/xfsutils/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xfsutils // import "yunion.io/x/onecloud/pkg/util/xfsutils"