Skip to content

Commit

Permalink
device: Add SCSI block device support
Browse files Browse the repository at this point in the history
Recently, the support for block devices (rootfs or support for
devices) plugged through the VM with virtio-blk have been added.

This commit follows this by allowing block devices plugged through the
VM using virtio-scsi to be supported and properly processed from the
agent.

One thing to notice here is that latest versions of Qemu (>= 2.9) have
some stability issues using virtio-blk, and the use of virtio-scsi is
a good way to work around this current limitation.

Fixes kata-containers#162

Signed-off-by: Sebastien Boeuf <[email protected]>
  • Loading branch information
Sebastien Boeuf committed Mar 14, 2018
1 parent 15e851f commit 9760c10
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 28 deletions.
153 changes: 129 additions & 24 deletions device.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
package main

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
Expand All @@ -22,34 +24,73 @@ import (
)

const (
driver9pType = "9p"
driverBlkType = "blk"
driver9pType = "9p"
driverBlkType = "blk"
driverSCSIType = "scsi"
)

// SCSI variables
var (
// Here in "0:0", the first number is the SCSI host number because
// only one SCSI controller has been plugged, while the second number
// is always 0.
scsiHostChannel = "0:0:"
sysClassPrefix = "/sys/class"
scsiDiskPrefix = filepath.Join(sysClassPrefix, "scsi_disk", scsiHostChannel)
scsiBlockSuffix = "block"
scsiDiskSuffix = filepath.Join("/device", scsiBlockSuffix)
scsiHostPath = filepath.Join(sysClassPrefix, "scsi_host")
)

type deviceHandler func(device pb.Device, spec *pb.Spec) error

var deviceHandlerList = map[string]deviceHandler{
driverBlkType: virtioBlkDeviceHandler,
driverBlkType: virtioBlkDeviceHandler,
driverSCSIType: virtioSCSIDeviceHandler,
}

func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error {
// First need to make sure the expected device shows up properly,
// and then we need to retrieve its device info (such as major and
// minor numbers), useful to update the device provided
// through the OCI specification.
if err := waitForDevice(device.VmPath); err != nil {
devName := strings.TrimPrefix(device.VmPath, devPrefix)
checkUevent := func(uEv *uevent.Uevent) bool {
return (uEv.Action == "add" &&
filepath.Base(uEv.DevPath) == devName)
}
if err := waitForDevice(device.VmPath, devName, checkUevent); err != nil {
return err
}

return updateSpecDeviceList(device, spec)
}

func virtioSCSIDeviceHandler(device pb.Device, spec *pb.Spec) error {
// Retrieve the device path from SCSI address.
devPath, err := getSCSIDevPath(device.Id)
if err != nil {
return err
}
device.VmPath = devPath

return updateSpecDeviceList(device, spec)
}

// updateSpecDeviceList takes a device description provided by the caller,
// trying to find it on the guest. Once this device has been identified, the
// "real" information that can be read from inside the VM is used to update
// the same device in the list of devices provided through the OCI spec.
// This is needed to update information about minor/major numbers that cannot
// be predicted from the caller.
func updateSpecDeviceList(device pb.Device, spec *pb.Spec) error {
// If no ContainerPath is provided, we won't be able to match and
// update the device in the OCI spec device list. This is an error.
if device.ContainerPath == "" {
return grpcStatus.Errorf(codes.Internal,
"ContainerPath cannot be empty")
}

// At this point in the code, we assume the specification will be
// updated, meaning we should make sure we have valid pointers here.
if spec.Linux == nil || len(spec.Linux.Devices) == 0 {
return grpcStatus.Errorf(codes.Internal,
"No devices found from the spec, cannot update")
Expand All @@ -72,7 +113,6 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error {
}).Info("handling block device")

// Update the spec
updated := false
for idx, d := range spec.Linux.Devices {
if d.Path == device.ContainerPath {
agentLog.WithFields(logrus.Fields{
Expand All @@ -84,27 +124,19 @@ func virtioBlkDeviceHandler(device pb.Device, spec *pb.Spec) error {
}).Info("updating block device major/minor into the spec")
spec.Linux.Devices[idx].Major = major
spec.Linux.Devices[idx].Minor = minor
updated = true
break
}
}

if !updated {
return grpcStatus.Errorf(codes.Internal,
"Should have found a matching device %s in the spec",
device.VmPath)
return nil
}
}

return nil
return grpcStatus.Errorf(codes.Internal,
"Should have found a matching device %s in the spec",
device.VmPath)
}

func waitForDevice(devicePath string) error {
deviceName := strings.TrimPrefix(devicePath, devPrefix)

if _, err := os.Stat(devicePath); err == nil {
return nil
}
type checkUeventCb func(uEv *uevent.Uevent) bool

func waitForDevice(devicePath, deviceName string, checkUevent checkUeventCb) error {
uEvHandler, err := uevent.NewHandler()
if err != nil {
return err
Expand Down Expand Up @@ -144,8 +176,7 @@ func waitForDevice(devicePath string) error {

fieldLogger.Info("Got uevent")

if uEv.Action == "add" &&
filepath.Base(uEv.DevPath) == deviceName {
if checkUevent(uEv) {
fieldLogger.Info("Hotplug event received")
break
}
Expand All @@ -165,6 +196,80 @@ func waitForDevice(devicePath string) error {
return nil
}

// scanSCSIBus scans SCSI bus for the given SCSI address(SCSI-Id and LUN)
func scanSCSIBus(scsiAddr string) error {
files, err := ioutil.ReadDir(scsiHostPath)
if err != nil {
return err
}

tokens := strings.Split(scsiAddr, ":")
if len(tokens) != 2 {
return grpcStatus.Errorf(codes.Internal,
"Unexpected format for SCSI Address : %s, expect SCSIID:LUN",
scsiAddr)
}

// Scan scsi host passing in the channel, SCSI id and LUN. Channel
// is always 0 because we have only one SCSI controller.
scanData := []byte(fmt.Sprintf("0 %s %s", tokens[0], tokens[1]))

for _, file := range files {
host := file.Name()
scanPath := filepath.Join(scsiHostPath, host, "scan")
if err := ioutil.WriteFile(scanPath, scanData, 0200); err != nil {
return err
}
}

return nil
}

// findSCSIDisk finds the SCSI disk name associated with the given SCSI path.
// This approach eliminates the need to predict the disk name on the host side,
// but we do need to rescan SCSI bus for this.
func findSCSIDisk(scsiPath string) (string, error) {
files, err := ioutil.ReadDir(scsiPath)
if err != nil {
return "", err
}

if len(files) != 1 {
return "", grpcStatus.Errorf(codes.Internal,
"Expecting a single SCSI device, found %v",
files)
}

return files[0].Name(), nil
}

// getSCSIDevPath scans SCSI bus looking for the provided SCSI address, then
// it waits for the SCSI disk to become available and returns the device path
// associated with the disk.
func getSCSIDevPath(scsiAddr string) (string, error) {
if err := scanSCSIBus(scsiAddr); err != nil {
return "", err
}

devPath := filepath.Join(scsiDiskPrefix+scsiAddr, scsiDiskSuffix)

checkUevent := func(uEv *uevent.Uevent) bool {
devSubPath := filepath.Join(scsiHostChannel+scsiAddr, scsiBlockSuffix)
return (uEv.Action == "add" &&
strings.Contains(uEv.DevPath, devSubPath))
}
if err := waitForDevice(devPath, scsiAddr, checkUevent); err != nil {
return "", err
}

scsiDiskName, err := findSCSIDisk(devPath)
if err != nil {
return "", err
}

return filepath.Join(devPrefix, scsiDiskName), nil
}

func addDevices(devices []*pb.Device, spec *pb.Spec) error {
for _, device := range devices {
if device == nil {
Expand Down
39 changes: 39 additions & 0 deletions device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"

pb "github.com/kata-containers/agent/protocols/grpc"
Expand Down Expand Up @@ -72,6 +73,44 @@ func TestVirtioBlkDeviceHandlerEmptyLinuxDevicesSpecFailure(t *testing.T) {
testVirtioBlkDeviceHandlerFailure(t, device, spec)
}

func TestScanSCSIBus(t *testing.T) {
testDir, err := ioutil.TempDir("", "kata-agent-tmp-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(testDir)

scsiHostPath = filepath.Join(testDir, "scsi_host")
os.RemoveAll(scsiHostPath)

defer os.RemoveAll(scsiHostPath)

scsiAddr := "1"

err = scanSCSIBus(scsiAddr)
assert.NotNil(t, err, "scanSCSIBus() should have failed")

if err := os.MkdirAll(scsiHostPath, mountPerm); err != nil {
t.Fatal(err)
}

scsiAddr = "1:1"
err = scanSCSIBus(scsiAddr)
assert.Nil(t, err, "scanSCSIBus() failed: %v", err)

host := filepath.Join(scsiHostPath, "host0")
if err := os.MkdirAll(host, mountPerm); err != nil {
t.Fatal(err)
}

err = scanSCSIBus(scsiAddr)
assert.Nil(t, err, "scanSCSIBus() failed: %v", err)

scanPath := filepath.Join(host, "scan")
_, err = os.Stat(scanPath)
assert.Nil(t, err, "os.Stat() %s failed: %v", scanPath, err)
}

func testAddDevicesSuccessful(t *testing.T, devices []*pb.Device, spec *pb.Spec) {
err := addDevices(devices, spec)
assert.Nil(t, err, "addDevices() failed: %v", err)
Expand Down
27 changes: 23 additions & 4 deletions mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"syscall"

"github.com/kata-containers/agent/pkg/uevent"
pb "github.com/kata-containers/agent/protocols/grpc"
"golang.org/x/sys/unix"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -157,10 +158,11 @@ func removeMounts(mounts []string) error {
// type of storage driver.
type storageHandler func(storage pb.Storage) (string, error)

// driverStorageHandlerList lists the supported drivers.
// storageHandlerList lists the supported drivers.
var storageHandlerList = map[string]storageHandler{
driver9pType: virtio9pStorageHandler,
driverBlkType: virtioBlkStorageHandler,
driver9pType: virtio9pStorageHandler,
driverBlkType: virtioBlkStorageHandler,
driverSCSIType: virtioSCSIStorageHandler,
}

// virtio9pStorageHandler handles the storage for 9p driver.
Expand All @@ -171,9 +173,26 @@ func virtio9pStorageHandler(storage pb.Storage) (string, error) {
// virtioBlkStorageHandler handles the storage for blk driver.
func virtioBlkStorageHandler(storage pb.Storage) (string, error) {
// First need to make sure the expected device shows up properly.
if err := waitForDevice(storage.Source); err != nil {
devName := strings.TrimPrefix(storage.Source, devPrefix)
checkUevent := func(uEv *uevent.Uevent) bool {
return (uEv.Action == "add" &&
filepath.Base(uEv.DevPath) == devName)
}
if err := waitForDevice(storage.Source, devName, checkUevent); err != nil {
return "", err
}

return commonStorageHandler(storage)
}

// virtioSCSIStorageHandler handles the storage for scsi driver.
func virtioSCSIStorageHandler(storage pb.Storage) (string, error) {
// Retrieve the device path from SCSI address.
devPath, err := getSCSIDevPath(storage.Source)
if err != nil {
return "", err
}
storage.Source = devPath

return commonStorageHandler(storage)
}
Expand Down

0 comments on commit 9760c10

Please sign in to comment.