diff --git a/device.go b/device.go index 84eb528101..41a076b412 100644 --- a/device.go +++ b/device.go @@ -7,6 +7,8 @@ package main import ( + "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -22,14 +24,29 @@ import ( ) const ( - driver9pType = "9p" - driverBlkType = "blk" + driver9pType = "9p" + driverBlkType = "blk" + driverSCSIType = "scsi" +) + +// SCSI variables +var ( + // Here in "0:0", the first number is the SCSI host number because + // only one SCSI controller has been plugged, while the second number + // is always 0. + scsiHostChannel = "0:0:" + sysClassPrefix = "/sys/class" + scsiDiskPrefix = filepath.Join(sysClassPrefix, "scsi_disk", scsiHostChannel) + scsiBlockSuffix = "block" + scsiDiskSuffix = filepath.Join("/device", scsiBlockSuffix) + scsiHostPath = filepath.Join(sysClassPrefix, "scsi_host") ) type deviceHandler func(device pb.Device, spec *pb.Spec) error var deviceHandlerList = map[string]deviceHandler{ - driverBlkType: virtioBlkDeviceHandler, + driverBlkType: virtioBlkDeviceHandler, + driverSCSIType: virtioSCSIDeviceHandler, } func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error { @@ -37,10 +54,36 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error { // and then we need to retrieve its device info (such as major and // minor numbers), useful to update the device provided // through the OCI specification. - if err := waitForDevice(device.VmPath); err != nil { + devName := strings.TrimPrefix(device.VmPath, devPrefix) + checkUevent := func(uEv *uevent.Uevent) bool { + return (uEv.Action == "add" && + filepath.Base(uEv.DevPath) == devName) + } + if err := waitForDevice(device.VmPath, devName, checkUevent); err != nil { + return err + } + + return updateSpecDeviceList(device, spec) +} + +func virtioSCSIDeviceHandler(device pb.Device, spec *pb.Spec) error { + // Retrieve the device path from SCSI address. + devPath, err := getSCSIDevPath(device.Id) + if err != nil { return err } + device.VmPath = devPath + + return updateSpecDeviceList(device, spec) +} +// updateSpecDeviceList takes a device description provided by the caller, +// trying to find it on the guest. Once this device has been identified, the +// "real" information that can be read from inside the VM is used to update +// the same device in the list of devices provided through the OCI spec. +// This is needed to update information about minor/major numbers that cannot +// be predicted from the caller. +func updateSpecDeviceList(device pb.Device, spec *pb.Spec) error { // If no ContainerPath is provided, we won't be able to match and // update the device in the OCI spec device list. This is an error. if device.ContainerPath == "" { @@ -48,8 +91,6 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error { "ContainerPath cannot be empty") } - // At this point in the code, we assume the specification will be - // updated, meaning we should make sure we have valid pointers here. if spec.Linux == nil || len(spec.Linux.Devices) == 0 { return grpcStatus.Errorf(codes.Internal, "No devices found from the spec, cannot update") @@ -72,7 +113,6 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error { }).Info("handling block device") // Update the spec - updated := false for idx, d := range spec.Linux.Devices { if d.Path == device.ContainerPath { agentLog.WithFields(logrus.Fields{ @@ -84,27 +124,19 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error { }).Info("updating block device major/minor into the spec") spec.Linux.Devices[idx].Major = major spec.Linux.Devices[idx].Minor = minor - updated = true - break - } - } - if !updated { - return grpcStatus.Errorf(codes.Internal, - "Should have found a matching device %s in the spec", - device.VmPath) + return nil + } } - return nil + return grpcStatus.Errorf(codes.Internal, + "Should have found a matching device %s in the spec", + device.VmPath) } -func waitForDevice(devicePath string) error { - deviceName := strings.TrimPrefix(devicePath, devPrefix) - - if _, err := os.Stat(devicePath); err == nil { - return nil - } +type checkUeventCb func(uEv *uevent.Uevent) bool +func waitForDevice(devicePath, deviceName string, checkUevent checkUeventCb) error { uEvHandler, err := uevent.NewHandler() if err != nil { return err @@ -144,8 +176,7 @@ func waitForDevice(devicePath string) error { fieldLogger.Info("Got uevent") - if uEv.Action == "add" && - filepath.Base(uEv.DevPath) == deviceName { + if checkUevent(uEv) { fieldLogger.Info("Hotplug event received") break } @@ -165,6 +196,80 @@ func waitForDevice(devicePath string) error { return nil } +// scanSCSIBus scans SCSI bus for the given SCSI address(SCSI-Id and LUN) +func scanSCSIBus(scsiAddr string) error { + files, err := ioutil.ReadDir(scsiHostPath) + if err != nil { + return err + } + + tokens := strings.Split(scsiAddr, ":") + if len(tokens) != 2 { + return grpcStatus.Errorf(codes.Internal, + "Unexpected format for SCSI Address : %s, expect SCSIID:LUN", + scsiAddr) + } + + // Scan scsi host passing in the channel, SCSI id and LUN. Channel + // is always 0 because we have only one SCSI controller. + scanData := []byte(fmt.Sprintf("0 %s %s", tokens[0], tokens[1])) + + for _, file := range files { + host := file.Name() + scanPath := filepath.Join(scsiHostPath, host, "scan") + if err := ioutil.WriteFile(scanPath, scanData, 0200); err != nil { + return err + } + } + + return nil +} + +// findSCSIDisk finds the SCSI disk name associated with the given SCSI path. +// This approach eliminates the need to predict the disk name on the host side, +// but we do need to rescan SCSI bus for this. +func findSCSIDisk(scsiPath string) (string, error) { + files, err := ioutil.ReadDir(scsiPath) + if err != nil { + return "", err + } + + if len(files) != 1 { + return "", grpcStatus.Errorf(codes.Internal, + "Expecting a single SCSI device, found %v", + files) + } + + return files[0].Name(), nil +} + +// getSCSIDevPath scans SCSI bus looking for the provided SCSI address, then +// it waits for the SCSI disk to become available and returns the device path +// associated with the disk. +func getSCSIDevPath(scsiAddr string) (string, error) { + if err := scanSCSIBus(scsiAddr); err != nil { + return "", err + } + + devPath := filepath.Join(scsiDiskPrefix+scsiAddr, scsiDiskSuffix) + + checkUevent := func(uEv *uevent.Uevent) bool { + devSubPath := filepath.Join(scsiHostChannel+scsiAddr, scsiBlockSuffix) + return (uEv.Action == "add" && + strings.Contains(uEv.DevPath, devSubPath)) + } + if err := waitForDevice(devPath, scsiAddr, checkUevent); err != nil { + return "", err + } + + scsiDiskName, err := findSCSIDisk(devPath) + if err != nil { + return "", err + } + + return filepath.Join(devPrefix, scsiDiskName), nil +} + func addDevices(devices []*pb.Device, spec *pb.Spec) error { for _, device := range devices { if device == nil { diff --git a/device_test.go b/device_test.go index 8249481aaf..323759ef26 100644 --- a/device_test.go +++ b/device_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "testing" pb "github.com/kata-containers/agent/protocols/grpc" @@ -72,6 +73,44 @@ func TestVirtioBlkDeviceHandlerEmptyLinuxDevicesSpecFailure(t *testing.T) { testVirtioBlkDeviceHandlerFailure(t, device, spec) } +func TestScanSCSIBus(t *testing.T) { + testDir, err := ioutil.TempDir("", "kata-agent-tmp-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(testDir) + + scsiHostPath = filepath.Join(testDir, "scsi_host") + os.RemoveAll(scsiHostPath) + + defer os.RemoveAll(scsiHostPath) + + scsiAddr := "1" + + err = scanSCSIBus(scsiAddr) + assert.NotNil(t, err, "scanSCSIBus() should have failed") + + if err := os.MkdirAll(scsiHostPath, mountPerm); err != nil { + t.Fatal(err) + } + + scsiAddr = "1:1" + err = scanSCSIBus(scsiAddr) + assert.Nil(t, err, "scanSCSIBus() failed: %v", err) + + host := filepath.Join(scsiHostPath, "host0") + if err := os.MkdirAll(host, mountPerm); err != nil { + t.Fatal(err) + } + + err = scanSCSIBus(scsiAddr) + assert.Nil(t, err, "scanSCSIBus() failed: %v", err) + + scanPath := filepath.Join(host, "scan") + _, err = os.Stat(scanPath) + assert.Nil(t, err, "os.Stat() %s failed: %v", scanPath, err) +} + func testAddDevicesSuccessful(t *testing.T, devices []*pb.Device, spec *pb.Spec) { err := addDevices(devices, spec) assert.Nil(t, err, "addDevices() failed: %v", err) diff --git a/mount.go b/mount.go index d11fa4bbd0..c1104c252f 100644 --- a/mount.go +++ b/mount.go @@ -12,6 +12,7 @@ import ( "strings" "syscall" + "github.com/kata-containers/agent/pkg/uevent" pb "github.com/kata-containers/agent/protocols/grpc" "golang.org/x/sys/unix" "google.golang.org/grpc/codes" @@ -157,10 +158,11 @@ func removeMounts(mounts []string) error { // type of storage driver. type storageHandler func(storage pb.Storage) (string, error) -// driverStorageHandlerList lists the supported drivers. +// storageHandlerList lists the supported drivers. var storageHandlerList = map[string]storageHandler{ - driver9pType: virtio9pStorageHandler, - driverBlkType: virtioBlkStorageHandler, + driver9pType: virtio9pStorageHandler, + driverBlkType: virtioBlkStorageHandler, + driverSCSIType: virtioSCSIStorageHandler, } // virtio9pStorageHandler handles the storage for 9p driver. @@ -171,9 +173,26 @@ func virtio9pStorageHandler(storage pb.Storage) (string, error) { // virtioBlkStorageHandler handles the storage for blk driver. func virtioBlkStorageHandler(storage pb.Storage) (string, error) { // First need to make sure the expected device shows up properly. - if err := waitForDevice(storage.Source); err != nil { + devName := strings.TrimPrefix(storage.Source, devPrefix) + checkUevent := func(uEv *uevent.Uevent) bool { + return (uEv.Action == "add" && + filepath.Base(uEv.DevPath) == devName) + } + if err := waitForDevice(storage.Source, devName, checkUevent); err != nil { + return "", err + } + + return commonStorageHandler(storage) +} + +// virtioSCSIStorageHandler handles the storage for scsi driver. +func virtioSCSIStorageHandler(storage pb.Storage) (string, error) { + // Retrieve the device path from SCSI address. + devPath, err := getSCSIDevPath(storage.Source) + if err != nil { return "", err } + storage.Source = devPath return commonStorageHandler(storage) }