fix(region,host): isolated device batch sync (#20537)

yunionio · Jun 16, 2024 · ddee083 · ddee083
1 parent 573fb2a
commit ddee083
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 18 deletions.
diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go
@@ -1767,6 +1767,13 @@ func reflectDispatcherInternal(
 	}
 }
 
+func DoUpdate(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	lockman.LockObject(ctx, item)
+	defer lockman.ReleaseObject(ctx, item)
+
+	return updateItem(manager, item, ctx, userCred, query, data)
+}
+
 func updateItem(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	var err error
 

diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go
@@ -2196,7 +2196,7 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc
 	host, _ := self.GetHost()
 	lockman.LockObject(ctx, host)
 	defer lockman.ReleaseObject(ctx, host)
-	devs, err := IsolatedDeviceManager.GetDevsOnHost(host.Id, devModel, count)
+	devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count)
 	if err != nil {
 		return httperrors.NewInternalServerError("fetch gpu failed %s", err)
 	}

diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go
@@ -6636,6 +6636,73 @@ func (hh *SHost) PerformProbeIsolatedDevices(ctx context.Context, userCred mccli
 	return driver.RequestProbeIsolatedDevices(ctx, userCred, hh, data)
 }
 
+func (hh *SHost) PerformSyncIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	devs, err := IsolatedDeviceManager.GetAllDevsOnHost(hh.Id)
+	if err != nil {
+		return nil, err
+	}
+	reqDevs, err := data.GetArray("isolated_devices")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("isolated_devices")
+	}
+
+	retDevs := jsonutils.NewArray()
+	foundDevIndex := map[int]struct{}{}
+	for i := range devs {
+		dev, err := IsolatedDeviceManager.FetchById(devs[i].Id)
+		if err != nil {
+			return nil, err
+		}
+
+		foundDev := false
+		for j := range reqDevs {
+			venderDeviceId, _ := reqDevs[j].GetString("vendor_device_id")
+			devAddr, _ := reqDevs[j].GetString("addr")
+			mdevId, _ := reqDevs[j].GetString("mdev_id")
+			if devs[i].VendorDeviceId == venderDeviceId && devs[i].Addr == devAddr && devs[i].MdevId == mdevId {
+				// update isolated device
+				devRet, err := db.DoUpdate(IsolatedDeviceManager, dev, ctx, userCred, jsonutils.NewDict(), reqDevs[j])
+				if err != nil {
+					return nil, err
+				}
+				retDevs.Add(devRet)
+				foundDevIndex[j] = struct{}{}
+				foundDev = true
+			}
+		}
+
+		if !foundDev {
+			// detach isolated device
+			isolatedDev := dev.(*SIsolatedDevice)
+			params := jsonutils.NewDict()
+			params.Set("purge", jsonutils.JSONTrue)
+			_, err := isolatedDev.PerformPurge(ctx, userCred, nil, params)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	for i := range reqDevs {
+		if _, ok := foundDevIndex[i]; ok {
+			continue
+		}
+		// create isolated device
+		dev, err := db.DoCreate(IsolatedDeviceManager, ctx, userCred, nil, reqDevs[i], userCred)
+		if err != nil {
+			return nil, err
+		}
+		devRet, err := db.GetItemDetails(IsolatedDeviceManager, dev, ctx, userCred)
+		if err != nil {
+			return nil, err
+		}
+		retDevs.Add(devRet)
+	}
+	res := jsonutils.NewDict()
+	res.Set("isolated_devices", retDevs)
+	return res, nil
+}
+
 func (hh *SHost) GetPinnedCpusetCores(ctx context.Context, userCred mcclient.TokenCredential) (map[string][]int, error) {
 	gsts, err := hh.GetGuests()
 	if err != nil {

diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go
@@ -1019,7 +1019,20 @@ func (manager *SIsolatedDeviceManager) DeleteDevicesByHost(ctx context.Context,
 	}
 }
 
-func (manager *SIsolatedDeviceManager) GetDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) {
+func (manager *SIsolatedDeviceManager) GetAllDevsOnHost(hostId string) ([]SIsolatedDevice, error) {
+	devs := make([]SIsolatedDevice, 0)
+	q := manager.Query().Equals("host_id", hostId)
+	err := db.FetchModelObjects(manager, q, &devs)
+	if err != nil {
+		return nil, err
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	return devs, nil
+}
+
+func (manager *SIsolatedDeviceManager) GetUnusedDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) {
 	devs := make([]SIsolatedDevice, 0)
 	q := manager.Query().Equals("host_id", hostId).Equals("model", model).IsNullOrEmpty("guest_id").Limit(count)
 	err := db.FetchModelObjects(manager, q, &devs)

diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go
@@ -2125,10 +2125,31 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		enableDevWhitelist,
 	)
 
-	objs, err := h.getRemoteIsolatedDevices()
+	h.IsolatedDeviceMan.BatchCustomProbe()
+	// sync each isolated device found
+	updateDevs := jsonutils.NewArray()
+	for _, dev := range h.IsolatedDeviceMan.GetDevices() {
+		dev.SetHostId(h.HostId)
+		data := isolated_device.GetApiResourceData(dev)
+		updateDevs.Add(data)
+	}
+
+	params := jsonutils.NewDict()
+	params.Set("isolated_devices", updateDevs)
+	ret, err := modules.Hosts.PerformAction(h.GetSession(), h.HostId, "sync-isolated-devices", params)
+	if err != nil {
+		return nil, errors.Wrap(err, "sync isolated devices")
+	}
+	devRet, err := ret.Get("isolated_devices")
 	if err != nil {
-		return nil, errors.Wrap(err, "getRemoteIsolatedDevices")
+		return nil, errors.Wrap(err, "sync isolated devices faild get dev rets")
 	}
+	devRets, _ := devRet.(*jsonutils.JSONArray)
+	if devRets.Length() != len(h.IsolatedDeviceMan.GetDevices()) {
+		return nil, errors.Wrap(err, "sync devices not match")
+	}
+
+	objs, _ := devRets.GetArray()
 	for _, obj := range objs {
 		info := isolated_device.CloudDeviceInfo{}
 		if err := obj.Unmarshal(&info); err != nil {
@@ -2138,23 +2159,11 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		if dev != nil {
 			dev.SetDeviceInfo(info)
 		} else {
-			// detach device
-			h.IsolatedDeviceMan.AppendDetachedDevice(&info)
+			return nil, errors.Wrapf(err, "unknown device %s", obj)
 		}
 	}
-	h.IsolatedDeviceMan.StartDetachTask()
-	h.IsolatedDeviceMan.BatchCustomProbe()
 
-	// sync each isolated device found
-	updateDevs := jsonutils.NewArray()
-	for _, dev := range h.IsolatedDeviceMan.GetDevices() {
-		if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil {
-			return nil, errors.Wrapf(err, "Sync device %s", dev)
-		} else {
-			updateDevs.Add(obj)
-		}
-	}
-	return updateDevs, nil
+	return devRets, nil
 }
 
 func (h *SHostInfo) deployAdminAuthorizedKeys() {