Merge pull request #18517 from wanyaoqi/automated-cherry-pick-of-#185…

…15-upstream-master

Automated cherry pick of #18515: fix(host): detect blk dev numqueues and nics(no id) pci addresses

pkg/hostman/guestman/guestman.go

@@ -1011,6 +1011,9 @@ func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{
 		ret.Set("migrate_certs", jsonutils.Marshal(certs))
 	}
 	if migParams.LiveMigrate {
+		if err = guest.syncVirtioDiskNumQueues(); err != nil {
+			return nil, errors.Wrap(err, "syncVirtioDiskNumQueues")
+		}
 		ret.Set("src_desc", jsonutils.Marshal(guest.Desc))
 	}
 	return ret, nil

pkg/hostman/guestman/pci.go

@@ -716,12 +716,58 @@ func (s *SKVMGuestInstance) ensurePciAddresses() error {
 	return nil
 }
 
+func (s *SKVMGuestInstance) getNetdevOfThePciAddress(qtree string, addr *desc.PCIAddr) string {
+	var slotFunc = fmt.Sprintf("addr = %02x.%x", addr.Slot, addr.Function)
+	var addressFound = false
+	var lines = strings.Split(strings.TrimSuffix(qtree, "\r\n"), "\\r\\n")
+	var currentIndentLevel = -1
+	for _, line := range lines {
+		trimmedLine := strings.TrimSpace(line)
+		if trimmedLine == "" {
+			continue
+		}
+
+		if currentIndentLevel > 0 {
+			newIndentLevel := len(line) - len(trimmedLine)
+			if newIndentLevel <= currentIndentLevel {
+				if addressFound {
+					break
+				}
+
+				currentIndentLevel = -1
+				continue
+			}
+
+			if strings.HasPrefix(trimmedLine, slotFunc) {
+				addressFound = true
+				continue
+			}
+
+			if strings.HasPrefix(trimmedLine, "netdev =") {
+				segs := strings.Split(trimmedLine, " ")
+				netdev := strings.Trim(segs[2], `\\"`)
+				log.Infof("found netdev %s: %s", netdev, trimmedLine)
+				return netdev
+			} else {
+				continue
+			}
+		}
+
+		if strings.HasPrefix(trimmedLine, "dev: virtio-net-pci") {
+			currentIndentLevel = len(line) - len(trimmedLine)
+		} else {
+			continue
+		}
+	}
+	return ""
+}
+
 // guests description no pci description before host-agent assign pci device address info
 // in this case wo need query pci address info by `query-pci` command. Also memory devices.
 func (s *SKVMGuestInstance) initGuestDescFromExistingGuest(
 	cpuList []monitor.HotpluggableCPU, pciInfoList []monitor.PCIInfo,
 	memoryDevicesInfoList []monitor.MemoryDeviceInfo, memDevs []monitor.Memdev,
-	scsiNumQueues int64,
+	scsiNumQueues int64, qtree string,
 ) error {
 	if len(pciInfoList) > 1 {
 		return errors.Errorf("unsupported pci info list with multi bus")
@@ -738,7 +784,8 @@ func (s *SKVMGuestInstance) initGuestDescFromExistingGuest(
 	}
 	s.initMachineDesc()
 
-	// TODO: pcie extend bus
+	// This code is designed to ensure compatibility with older guests.
+	// However, it is not recommended for new guests to generate a desc file from it
 	pciRoot, _ := s.initGuestPciControllers(false)
 	err = s.initGuestPciAddresses()
 	if err != nil {
@@ -977,6 +1024,24 @@ func (s *SKVMGuestInstance) initGuestDescFromExistingGuest(
 					if err != nil {
 						return errors.Wrap(err, "ensure vga pci address")
 					}
+				case class == 512 && vendor == 6900 && device == 4096: // { 0x0200, "Ethernet controller", "ethernet"}, 1af4:1000  network device (legacy)
+					// virtio nics has no ids
+					ifname := s.getNetdevOfThePciAddress(qtree, pciAddr)
+
+					index := 0
+					for ; index < len(s.Desc.Nics); index++ {
+						if s.Desc.Nics[index].Ifname == ifname {
+							s.Desc.Nics[index].Pci.PCIAddr = pciAddr
+							err = s.ensureDevicePciAddress(s.Desc.Nics[index].Pci, -1, nil)
+							if err != nil {
+								return errors.Wrapf(err, "ensure nic %s pci address", s.Desc.Nics[index].Ifname)
+							}
+							break
+						}
+					}
+					if index >= len(s.Desc.Nics) {
+						return errors.Errorf("failed find nics ifname")
+					}
 				default:
 					unknownDevices = append(unknownDevices, pciInfoList[0].Devices[i])
 				}

pkg/hostman/guestman/qemu-kvm.go

@@ -457,7 +457,15 @@ func (s *SKVMGuestInstance) LoadDesc() error {
 
 	if s.IsRunning() {
 		if len(s.Desc.PCIControllers) > 0 {
-			return s.loadGuestPciAddresses()
+			if err := s.loadGuestPciAddresses(); err != nil {
+				log.Errorf("failed load guest %s pci addresses %s", s.GetName(), err)
+				if len(s.Desc.AnonymousPCIDevs) > 0 {
+					s.Desc = s.SourceDesc
+					s.pciUninitialized = true
+				} else {
+					return err
+				}
+			}
 		} else {
 			s.pciUninitialized = true
 		}
@@ -1115,17 +1123,49 @@ func (s *SKVMGuestInstance) collectGuestDescription() error {
 	if err != nil {
 		return errors.Wrap(err, "query mem devs")
 	}
-	scsiNumQueues := s.getScsiNumQueues()
-	err = s.initGuestDescFromExistingGuest(cpuList, pciInfoList, memoryDevicesInfoList, memDevs, scsiNumQueues)
+
+	qtree := s.infoQtree()
+	scsiNumQueues := s.getScsiNumQueues(qtree)
+	for i := range s.Desc.Disks {
+		// fix virtio disk driver num-queues
+		if s.Desc.Disks[i].Driver == "virtio" {
+			diskDriver := fmt.Sprintf("drive_%d", s.Desc.Disks[i].Index)
+			numQueues := s.getDiskDriverNumQueues(qtree, diskDriver)
+			log.Infof("disk driver %s num queues %d", diskDriver, numQueues)
+			if numQueues > 0 {
+				s.Desc.Disks[i].NumQueues = uint8(numQueues)
+			}
+		}
+	}
+
+	err = s.initGuestDescFromExistingGuest(
+		cpuList, pciInfoList, memoryDevicesInfoList, memDevs, scsiNumQueues, qtree)
 	if err != nil {
 		return errors.Wrap(err, "failed init guest devices")
 	}
+
 	if err := s.SaveLiveDesc(s.Desc); err != nil {
 		return errors.Wrap(err, "failed save live desc")
 	}
 	return nil
 }
 
+func (s *SKVMGuestInstance) syncVirtioDiskNumQueues() error {
+	qtree := s.infoQtree()
+	for i := range s.Desc.Disks {
+		// fix virtio disk driver num-queues
+		if s.Desc.Disks[i].Driver == "virtio" {
+			diskDriver := fmt.Sprintf("drive_%d", s.Desc.Disks[i].Index)
+			numQueues := s.getDiskDriverNumQueues(qtree, diskDriver)
+			log.Infof("disk driver %s num queues %d", diskDriver, numQueues)
+			if numQueues > 0 {
+				s.Desc.Disks[i].NumQueues = uint8(numQueues)
+			}
+		}
+	}
+	return s.SaveLiveDesc(s.Desc)
+}
+
 func (s *SKVMGuestInstance) getHotpluggableCPUList() ([]monitor.HotpluggableCPU, error) {
 	var res []monitor.HotpluggableCPU
 	var errChan = make(chan error)
@@ -1142,13 +1182,79 @@ func (s *SKVMGuestInstance) getHotpluggableCPUList() ([]monitor.HotpluggableCPU,
 	return res, err
 }
 
-func (s *SKVMGuestInstance) getScsiNumQueues() int64 {
-	var numQueueChan = make(chan int64)
-	cb := func(numQueues int64) {
-		numQueueChan <- numQueues
+func (s *SKVMGuestInstance) infoQtree() string {
+	var qtreeChan = make(chan string)
+	cb := func(qtree string) {
+		qtreeChan <- qtree
 	}
-	s.Monitor.GetScsiNumQueues(cb)
-	return <-numQueueChan
+	s.Monitor.InfoQtree(cb)
+	return <-qtreeChan
+}
+
+func getScsiNumQueuesQmp(output string) int64 {
+	// if output is response from hmp, sep should use \r\n
+	var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\\r\\n")
+	for i, line := range lines {
+		line := strings.TrimSpace(line)
+		if strings.HasPrefix(line, "dev: virtio-scsi-device") {
+			if len(lines) <= i+1 {
+				log.Errorf("failed parse num queues")
+				return -1
+			}
+			line = strings.TrimSpace(lines[i+1])
+			segs := strings.Split(line, " ")
+			numQueue, err := strconv.ParseInt(segs[2], 10, 0)
+			if err != nil {
+				log.Errorf("failed parse num queue %s: %s", line, err)
+				return -1
+			} else {
+				return numQueue
+			}
+		}
+	}
+	return -1
+}
+
+func (s *SKVMGuestInstance) getScsiNumQueues(qtree string) int64 {
+	return getScsiNumQueuesQmp(qtree)
+}
+
+func (s *SKVMGuestInstance) getDiskDriverNumQueues(qtree, driver string) int64 {
+	var lines = strings.Split(strings.TrimSuffix(qtree, "\r\n"), "\\r\\n")
+	var currentIndentLevel = -1
+	for _, line := range lines {
+		trimmedLine := strings.TrimSpace(line)
+		if trimmedLine == "" {
+			continue
+		}
+
+		if currentIndentLevel > 0 {
+			// disk has been found
+			newIndentLevel := len(line) - len(trimmedLine)
+			if newIndentLevel <= currentIndentLevel { // run out disk driver block
+				break
+			}
+			if strings.HasPrefix(trimmedLine, "num-queues =") {
+				segs := strings.Split(trimmedLine, " ")
+				numQueue, err := strconv.ParseInt(segs[2], 10, 0)
+				if err != nil {
+					log.Errorf("failed parse num queue %s: %s", trimmedLine, err)
+					return -1
+				} else {
+					return numQueue
+				}
+			} else {
+				continue
+			}
+		}
+
+		if strings.HasPrefix(trimmedLine, "dev: virtio-blk-pci") && strings.Contains(line, driver) {
+			currentIndentLevel = len(line) - len(trimmedLine)
+		} else {
+			continue
+		}
+	}
+	return -1
 }
 
 func (s *SKVMGuestInstance) getPciDevices() ([]monitor.PCIInfo, error) {

pkg/hostman/guestman/qemu/generate.go

@@ -324,10 +324,6 @@ func getDiskDeviceOption(optDrv QemuOptions, disk *desc.SGuestDisk) string {
 	numQueues := disk.NumQueues
 	isSsd := disk.IsSSD
 
-	if numQueues == 0 {
-		numQueues = 4
-	}
-
 	var opt = ""
 	opt += GetDiskDeviceModel(diskDriver)
 	opt += fmt.Sprintf(",drive=drive_%d", diskIndex)
@@ -338,6 +334,9 @@ func getDiskDeviceOption(optDrv QemuOptions, disk *desc.SGuestDisk) string {
 		}
 		// opt += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1)
 		opt += ",iothread=iothread0"
+		if numQueues > 0 {
+			opt += fmt.Sprintf(",num-queues=%d,vectors=%d", numQueues, numQueues+1)
+		}
 	} else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
 		opt += ",bus=scsi.0"
 	} else if diskDriver == DISK_DRIVER_IDE {

pkg/hostman/monitor/hmp.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io"
 	"regexp"
-	"strconv"
 	"strings"
 	"time"
 
@@ -572,34 +571,7 @@ func (m *HmpMonitor) Quit(cb StringCallback) {
 	m.Query("quit", cb)
 }
 
-func getScsiNumQueues(output string) int64 {
-	var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\r\n")
-	for i, line := range lines {
-		line := strings.TrimSpace(line)
-		if strings.HasPrefix(line, "dev: virtio-scsi-device") {
-			if len(lines) <= i+1 {
-				log.Errorf("failed parse num queues")
-				return -1
-			}
-			line = strings.TrimSpace(lines[i+1])
-			segs := strings.Split(line, " ")
-			numQueue, err := strconv.ParseInt(segs[2], 10, 0)
-			if err != nil {
-				log.Errorf("failed parse num queue %s", err)
-				return -1
-			} else {
-				return numQueue
-			}
-		}
-	}
-	return -1
-}
-
-func (m *HmpMonitor) GetScsiNumQueues(callback func(int64)) {
-	cb := func(output string) {
-		numQueues := getScsiNumQueues(output)
-		callback(numQueues)
-	}
+func (m *HmpMonitor) InfoQtree(cb StringCallback) {
 	m.Query("info qtree", cb)
 }
 

pkg/hostman/monitor/monitor.go

@@ -200,7 +200,7 @@ type Monitor interface {
 	GetBlockJobCounts(func(jobs int))
 	GetBlockJobs(func([]BlockJob))
 	QueryPci(callback QueryPciCallback)
-	GetScsiNumQueues(callback func(int64))
+	InfoQtree(cb StringCallback)
 
 	GetCpuCount(func(count int))
 	AddCpu(cpuIndex int, callback StringCallback)

pkg/hostman/monitor/qmp.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"regexp"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"time"
 
@@ -1105,34 +1104,7 @@ func (m *QmpMonitor) Quit(callback StringCallback) {
 	m.Query(cmd, cb)
 }
 
-func getScsiNumQueuesQmp(output string) int64 {
-	var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\\r\\n")
-	for i, line := range lines {
-		line := strings.TrimSpace(line)
-		if strings.HasPrefix(line, "dev: virtio-scsi-device") {
-			if len(lines) <= i+1 {
-				log.Errorf("failed parse num queues")
-				return -1
-			}
-			line = strings.TrimSpace(lines[i+1])
-			segs := strings.Split(line, " ")
-			numQueue, err := strconv.ParseInt(segs[2], 10, 0)
-			if err != nil {
-				log.Errorf("failed parse num queue %s", err)
-				return -1
-			} else {
-				return numQueue
-			}
-		}
-	}
-	return -1
-}
-
-func (m *QmpMonitor) GetScsiNumQueues(callback func(int64)) {
-	cb := func(output string) {
-		numQueues := getScsiNumQueuesQmp(output)
-		callback(numQueues)
-	}
+func (m *QmpMonitor) InfoQtree(cb StringCallback) {
 	m.HumanMonitorCommand("info qtree", cb)
 }