From ba4437434b99a8dd6883e6904b738ca0f5c04fd5 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Sun, 29 Oct 2023 20:42:14 +0800 Subject: [PATCH] fix(host): detect blk dev numqueues and nics(no id) pci addresses --- pkg/hostman/guestman/guestman.go | 3 + pkg/hostman/guestman/pci.go | 69 +++++++++++++- pkg/hostman/guestman/qemu-kvm.go | 124 ++++++++++++++++++++++++-- pkg/hostman/guestman/qemu/generate.go | 7 +- pkg/hostman/monitor/hmp.go | 30 +------ pkg/hostman/monitor/monitor.go | 2 +- pkg/hostman/monitor/qmp.go | 30 +------ 7 files changed, 191 insertions(+), 74 deletions(-) diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 72f1a846c38..f5fde8016c2 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -1011,6 +1011,9 @@ func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{ ret.Set("migrate_certs", jsonutils.Marshal(certs)) } if migParams.LiveMigrate { + if err = guest.syncVirtioDiskNumQueues(); err != nil { + return nil, errors.Wrap(err, "syncVirtioDiskNumQueues") + } ret.Set("src_desc", jsonutils.Marshal(guest.Desc)) } return ret, nil diff --git a/pkg/hostman/guestman/pci.go b/pkg/hostman/guestman/pci.go index 6871c1efa70..c17a376f8b9 100644 --- a/pkg/hostman/guestman/pci.go +++ b/pkg/hostman/guestman/pci.go @@ -716,12 +716,58 @@ func (s *SKVMGuestInstance) ensurePciAddresses() error { return nil } +func (s *SKVMGuestInstance) getNetdevOfThePciAddress(qtree string, addr *desc.PCIAddr) string { + var slotFunc = fmt.Sprintf("addr = %02x.%x", addr.Slot, addr.Function) + var addressFound = false + var lines = strings.Split(strings.TrimSuffix(qtree, "\r\n"), "\\r\\n") + var currentIndentLevel = -1 + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" { + continue + } + + if currentIndentLevel > 0 { + newIndentLevel := len(line) - len(trimmedLine) + if newIndentLevel <= currentIndentLevel { + if addressFound { + break + } + + currentIndentLevel = -1 + continue + } + + if strings.HasPrefix(trimmedLine, slotFunc) { + addressFound = true + continue + } + + if strings.HasPrefix(trimmedLine, "netdev =") { + segs := strings.Split(trimmedLine, " ") + netdev := strings.Trim(segs[2], `\\"`) + log.Infof("found netdev %s: %s", netdev, trimmedLine) + return netdev + } else { + continue + } + } + + if strings.HasPrefix(trimmedLine, "dev: virtio-net-pci") { + currentIndentLevel = len(line) - len(trimmedLine) + } else { + continue + } + } + return "" +} + // guests description no pci description before host-agent assign pci device address info // in this case wo need query pci address info by `query-pci` command. Also memory devices. func (s *SKVMGuestInstance) initGuestDescFromExistingGuest( cpuList []monitor.HotpluggableCPU, pciInfoList []monitor.PCIInfo, memoryDevicesInfoList []monitor.MemoryDeviceInfo, memDevs []monitor.Memdev, - scsiNumQueues int64, + scsiNumQueues int64, qtree string, ) error { if len(pciInfoList) > 1 { return errors.Errorf("unsupported pci info list with multi bus") @@ -738,7 +784,8 @@ func (s *SKVMGuestInstance) initGuestDescFromExistingGuest( } s.initMachineDesc() - // TODO: pcie extend bus + // This code is designed to ensure compatibility with older guests. + // However, it is not recommended for new guests to generate a desc file from it pciRoot, _ := s.initGuestPciControllers(false) err = s.initGuestPciAddresses() if err != nil { @@ -977,6 +1024,24 @@ func (s *SKVMGuestInstance) initGuestDescFromExistingGuest( if err != nil { return errors.Wrap(err, "ensure vga pci address") } + case class == 512 && vendor == 6900 && device == 4096: // { 0x0200, "Ethernet controller", "ethernet"}, 1af4:1000 network device (legacy) + // virtio nics has no ids + ifname := s.getNetdevOfThePciAddress(qtree, pciAddr) + + index := 0 + for ; index < len(s.Desc.Nics); index++ { + if s.Desc.Nics[index].Ifname == ifname { + s.Desc.Nics[index].Pci.PCIAddr = pciAddr + err = s.ensureDevicePciAddress(s.Desc.Nics[index].Pci, -1, nil) + if err != nil { + return errors.Wrapf(err, "ensure nic %s pci address", s.Desc.Nics[index].Ifname) + } + break + } + } + if index >= len(s.Desc.Nics) { + return errors.Errorf("failed find nics ifname") + } default: unknownDevices = append(unknownDevices, pciInfoList[0].Devices[i]) } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 18b7370417d..626e4aad46d 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -457,7 +457,15 @@ func (s *SKVMGuestInstance) LoadDesc() error { if s.IsRunning() { if len(s.Desc.PCIControllers) > 0 { - return s.loadGuestPciAddresses() + if err := s.loadGuestPciAddresses(); err != nil { + log.Errorf("failed load guest %s pci addresses %s", s.GetName(), err) + if len(s.Desc.AnonymousPCIDevs) > 0 { + s.Desc = s.SourceDesc + s.pciUninitialized = true + } else { + return err + } + } } else { s.pciUninitialized = true } @@ -1115,17 +1123,49 @@ func (s *SKVMGuestInstance) collectGuestDescription() error { if err != nil { return errors.Wrap(err, "query mem devs") } - scsiNumQueues := s.getScsiNumQueues() - err = s.initGuestDescFromExistingGuest(cpuList, pciInfoList, memoryDevicesInfoList, memDevs, scsiNumQueues) + + qtree := s.infoQtree() + scsiNumQueues := s.getScsiNumQueues(qtree) + for i := range s.Desc.Disks { + // fix virtio disk driver num-queues + if s.Desc.Disks[i].Driver == "virtio" { + diskDriver := fmt.Sprintf("drive_%d", s.Desc.Disks[i].Index) + numQueues := s.getDiskDriverNumQueues(qtree, diskDriver) + log.Infof("disk driver %s num queues %d", diskDriver, numQueues) + if numQueues > 0 { + s.Desc.Disks[i].NumQueues = uint8(numQueues) + } + } + } + + err = s.initGuestDescFromExistingGuest( + cpuList, pciInfoList, memoryDevicesInfoList, memDevs, scsiNumQueues, qtree) if err != nil { return errors.Wrap(err, "failed init guest devices") } + if err := s.SaveLiveDesc(s.Desc); err != nil { return errors.Wrap(err, "failed save live desc") } return nil } +func (s *SKVMGuestInstance) syncVirtioDiskNumQueues() error { + qtree := s.infoQtree() + for i := range s.Desc.Disks { + // fix virtio disk driver num-queues + if s.Desc.Disks[i].Driver == "virtio" { + diskDriver := fmt.Sprintf("drive_%d", s.Desc.Disks[i].Index) + numQueues := s.getDiskDriverNumQueues(qtree, diskDriver) + log.Infof("disk driver %s num queues %d", diskDriver, numQueues) + if numQueues > 0 { + s.Desc.Disks[i].NumQueues = uint8(numQueues) + } + } + } + return s.SaveLiveDesc(s.Desc) +} + func (s *SKVMGuestInstance) getHotpluggableCPUList() ([]monitor.HotpluggableCPU, error) { var res []monitor.HotpluggableCPU var errChan = make(chan error) @@ -1142,13 +1182,79 @@ func (s *SKVMGuestInstance) getHotpluggableCPUList() ([]monitor.HotpluggableCPU, return res, err } -func (s *SKVMGuestInstance) getScsiNumQueues() int64 { - var numQueueChan = make(chan int64) - cb := func(numQueues int64) { - numQueueChan <- numQueues +func (s *SKVMGuestInstance) infoQtree() string { + var qtreeChan = make(chan string) + cb := func(qtree string) { + qtreeChan <- qtree } - s.Monitor.GetScsiNumQueues(cb) - return <-numQueueChan + s.Monitor.InfoQtree(cb) + return <-qtreeChan +} + +func getScsiNumQueuesQmp(output string) int64 { + // if output is response from hmp, sep should use \r\n + var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\\r\\n") + for i, line := range lines { + line := strings.TrimSpace(line) + if strings.HasPrefix(line, "dev: virtio-scsi-device") { + if len(lines) <= i+1 { + log.Errorf("failed parse num queues") + return -1 + } + line = strings.TrimSpace(lines[i+1]) + segs := strings.Split(line, " ") + numQueue, err := strconv.ParseInt(segs[2], 10, 0) + if err != nil { + log.Errorf("failed parse num queue %s: %s", line, err) + return -1 + } else { + return numQueue + } + } + } + return -1 +} + +func (s *SKVMGuestInstance) getScsiNumQueues(qtree string) int64 { + return getScsiNumQueuesQmp(qtree) +} + +func (s *SKVMGuestInstance) getDiskDriverNumQueues(qtree, driver string) int64 { + var lines = strings.Split(strings.TrimSuffix(qtree, "\r\n"), "\\r\\n") + var currentIndentLevel = -1 + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" { + continue + } + + if currentIndentLevel > 0 { + // disk has been found + newIndentLevel := len(line) - len(trimmedLine) + if newIndentLevel <= currentIndentLevel { // run out disk driver block + break + } + if strings.HasPrefix(trimmedLine, "num-queues =") { + segs := strings.Split(trimmedLine, " ") + numQueue, err := strconv.ParseInt(segs[2], 10, 0) + if err != nil { + log.Errorf("failed parse num queue %s: %s", trimmedLine, err) + return -1 + } else { + return numQueue + } + } else { + continue + } + } + + if strings.HasPrefix(trimmedLine, "dev: virtio-blk-pci") && strings.Contains(line, driver) { + currentIndentLevel = len(line) - len(trimmedLine) + } else { + continue + } + } + return -1 } func (s *SKVMGuestInstance) getPciDevices() ([]monitor.PCIInfo, error) { diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index eda35da4be9..f294d54e545 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -324,10 +324,6 @@ func getDiskDeviceOption(optDrv QemuOptions, disk *desc.SGuestDisk) string { numQueues := disk.NumQueues isSsd := disk.IsSSD - if numQueues == 0 { - numQueues = 4 - } - var opt = "" opt += GetDiskDeviceModel(diskDriver) opt += fmt.Sprintf(",drive=drive_%d", diskIndex) @@ -338,6 +334,9 @@ func getDiskDeviceOption(optDrv QemuOptions, disk *desc.SGuestDisk) string { } // opt += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1) opt += ",iothread=iothread0" + if numQueues > 0 { + opt += fmt.Sprintf(",num-queues=%d,vectors=%d", numQueues, numQueues+1) + } } else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) { opt += ",bus=scsi.0" } else if diskDriver == DISK_DRIVER_IDE { diff --git a/pkg/hostman/monitor/hmp.go b/pkg/hostman/monitor/hmp.go index 9f8f3194e11..c9e87430600 100644 --- a/pkg/hostman/monitor/hmp.go +++ b/pkg/hostman/monitor/hmp.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "regexp" - "strconv" "strings" "time" @@ -572,34 +571,7 @@ func (m *HmpMonitor) Quit(cb StringCallback) { m.Query("quit", cb) } -func getScsiNumQueues(output string) int64 { - var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\r\n") - for i, line := range lines { - line := strings.TrimSpace(line) - if strings.HasPrefix(line, "dev: virtio-scsi-device") { - if len(lines) <= i+1 { - log.Errorf("failed parse num queues") - return -1 - } - line = strings.TrimSpace(lines[i+1]) - segs := strings.Split(line, " ") - numQueue, err := strconv.ParseInt(segs[2], 10, 0) - if err != nil { - log.Errorf("failed parse num queue %s", err) - return -1 - } else { - return numQueue - } - } - } - return -1 -} - -func (m *HmpMonitor) GetScsiNumQueues(callback func(int64)) { - cb := func(output string) { - numQueues := getScsiNumQueues(output) - callback(numQueues) - } +func (m *HmpMonitor) InfoQtree(cb StringCallback) { m.Query("info qtree", cb) } diff --git a/pkg/hostman/monitor/monitor.go b/pkg/hostman/monitor/monitor.go index aab120663e7..b09d27adf0b 100644 --- a/pkg/hostman/monitor/monitor.go +++ b/pkg/hostman/monitor/monitor.go @@ -200,7 +200,7 @@ type Monitor interface { GetBlockJobCounts(func(jobs int)) GetBlockJobs(func([]BlockJob)) QueryPci(callback QueryPciCallback) - GetScsiNumQueues(callback func(int64)) + InfoQtree(cb StringCallback) GetCpuCount(func(count int)) AddCpu(cpuIndex int, callback StringCallback) diff --git a/pkg/hostman/monitor/qmp.go b/pkg/hostman/monitor/qmp.go index 94c068c76af..f77b85fda55 100644 --- a/pkg/hostman/monitor/qmp.go +++ b/pkg/hostman/monitor/qmp.go @@ -21,7 +21,6 @@ import ( "io" "regexp" "runtime/debug" - "strconv" "strings" "time" @@ -1105,34 +1104,7 @@ func (m *QmpMonitor) Quit(callback StringCallback) { m.Query(cmd, cb) } -func getScsiNumQueuesQmp(output string) int64 { - var lines = strings.Split(strings.TrimSuffix(output, "\r\n"), "\\r\\n") - for i, line := range lines { - line := strings.TrimSpace(line) - if strings.HasPrefix(line, "dev: virtio-scsi-device") { - if len(lines) <= i+1 { - log.Errorf("failed parse num queues") - return -1 - } - line = strings.TrimSpace(lines[i+1]) - segs := strings.Split(line, " ") - numQueue, err := strconv.ParseInt(segs[2], 10, 0) - if err != nil { - log.Errorf("failed parse num queue %s", err) - return -1 - } else { - return numQueue - } - } - } - return -1 -} - -func (m *QmpMonitor) GetScsiNumQueues(callback func(int64)) { - cb := func(output string) { - numQueues := getScsiNumQueuesQmp(output) - callback(numQueues) - } +func (m *QmpMonitor) InfoQtree(cb StringCallback) { m.HumanMonitorCommand("info qtree", cb) }