Skip to content

Commit

Permalink
Merge pull request #18967 from wanyaoqi/automated-cherry-pick-of-#189…
Browse files Browse the repository at this point in the history 
…65-upstream-master

Automated cherry pick of #18965: fix(host-deployer): run guest on host and ignore no config file
  • Loading branch information
@zexi
zexi authored Dec 12, 2023
2 parents e8b5c4f + 1e0c76b commit 9052d5e
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 30 deletions.
21 changes: 17 additions & 4 deletions pkg/cloudcommon/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/atexit"
"yunion.io/x/onecloud/pkg/util/fileutils2"
)

const (
Expand Down Expand Up @@ -267,7 +268,15 @@ func (opt *DBOptions) GetClickhouseConnStr() (string, string, error) {
return "clickhouse", opt.Clickhouse, nil
}

func ParseOptionsIgnoreNoConfigfile(optStruct interface{}, args []string, configFileName string, serviceType string) {
parseOptions(optStruct, args, configFileName, serviceType, true)
}

func ParseOptions(optStruct interface{}, args []string, configFileName string, serviceType string) {
parseOptions(optStruct, args, configFileName, serviceType, false)
}

func parseOptions(optStruct interface{}, args []string, configFileName string, serviceType string, ignoreNoConfigfile bool) {
if len(serviceType) == 0 {
log.Fatalf("ServiceType must provided!")
}
Expand Down Expand Up @@ -317,10 +326,14 @@ func ParseOptions(optStruct interface{}, args []string, configFileName string, s
}

if len(optionsRef.Config) > 0 {
log.Infof("Use configuration file: %s", optionsRef.Config)
err = parser.ParseFile(optionsRef.Config)
if err != nil {
log.Fatalf("Parse configuration file: %v", err)
if !fileutils2.Exists(optionsRef.Config) && !ignoreNoConfigfile {
log.Fatalf("Configuration file %s not exist", optionsRef.Config)
} else if fileutils2.Exists(optionsRef.Config) {
log.Infof("Use configuration file: %s", optionsRef.Config)
err = parser.ParseFile(optionsRef.Config)
if err != nil {
log.Fatalf("Parse configuration file: %v", err)
}
}
}

Expand Down
144 changes: 128 additions & 16 deletions pkg/hostman/diskutils/qemu_kvm/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package qemu_kvm
import (
"encoding/json"
"fmt"
"io/ioutil"
"path"
"strings"
"sync"
"sync/atomic"
Expand All @@ -32,6 +34,7 @@ import (
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
"yunion.io/x/onecloud/pkg/util/qemuimg"
"yunion.io/x/onecloud/pkg/util/qemutils"
"yunion.io/x/onecloud/pkg/util/ssh"
"yunion.io/x/onecloud/pkg/util/sysutils"
)
Expand All @@ -47,7 +50,9 @@ var (
X86_INITRD_PATH = "/yunionos/x86_64/initramfs"
X86_KERNEL_PATH = "/yunionos/x86_64/kernel"

DEPLOY_ISO = "/opt/cloud/host_deployer_v1.iso"
RUN_ON_HOST_ROOT_PATH = "/opt/cloud/host-deployer"

DEPLOY_ISO = "/opt/cloud/host-deployer/host_deployer_v1.iso"
DEPLOYER_BIN = "/opt/yunion/bin/host-deployer"
YUNIONOS_PASSWD = "mosbaremetal"
)
Expand All @@ -58,10 +63,47 @@ type QemuDeployManager struct {
hugepageSizeKB int
portsInUse *sync.Map
lastUsedSshPort int
qemuCmd string

c chan struct{}
}

func (m *QemuDeployManager) runOnHost() bool {
return m.qemuCmd != QEMU_KVM_PATH
}

func (m *QemuDeployManager) GetX86InitrdPath() string {
if m.runOnHost() {
return path.Join(RUN_ON_HOST_ROOT_PATH, X86_INITRD_PATH)
} else {
return X86_INITRD_PATH
}
}

func (m *QemuDeployManager) GetARMInitrdPath() string {
if m.runOnHost() {
return path.Join(RUN_ON_HOST_ROOT_PATH, ARM_INITRD_PATH)
} else {
return ARM_INITRD_PATH
}
}

func (m *QemuDeployManager) GetX86KernelPath() string {
if m.runOnHost() {
return path.Join(RUN_ON_HOST_ROOT_PATH, X86_KERNEL_PATH)
} else {
return X86_KERNEL_PATH
}
}

func (m *QemuDeployManager) GetARMKernelPath() string {
if m.runOnHost() {
return path.Join(RUN_ON_HOST_ROOT_PATH, ARM_KERNEL_PATH)
} else {
return ARM_KERNEL_PATH
}
}

func (m *QemuDeployManager) Acquire() {
log.Infof("acquire QemuDeployManager")
m.c <- struct{}{}
Expand Down Expand Up @@ -107,11 +149,41 @@ func (m *QemuDeployManager) GetSshFreePort() int {

var manager *QemuDeployManager

func InitQemuDeployManager(cpuArch string, hugepage bool, hugepageSizeKB int, deployConcurrent int) error {
func tryCleanGuest(hmpPath string) {
var c = make(chan error)
onMonitorConnected := func() {
log.Infof("%s monitor connected", hmpPath)
c <- nil
}
onMonitorDisConnect := func(e error) {
log.Errorf("%s monitor disconnect %s", hmpPath, e)
}
onMonitorConnectFailed := func(e error) {
log.Errorf("%s monitor connect failed %s", hmpPath, e)
c <- e
}
m := monitor.NewHmpMonitor("", "", onMonitorDisConnect, onMonitorConnectFailed, onMonitorConnected)
if err := m.ConnectWithSocket(hmpPath); err != nil {
log.Errorf("failed connect socket %s %s", hmpPath, err)
} else {
<-c
m.Quit(func(string) {})
}
}

func InitQemuDeployManager(cpuArch, qemuVersion string, hugepage bool, hugepageSizeKB int, deployConcurrent int) error {
if deployConcurrent <= 0 {
deployConcurrent = 10
}

if cpuArch == OS_ARCH_AARCH64 {
qemutils.UseAarch64()
}
qemuCmd := qemutils.GetQemu(qemuVersion)
if qemuCmd == "" {
qemuCmd = QEMU_KVM_PATH
}

err := procutils.NewCommand("mkdir", "-p", "/etc/ceph").Run()
if err != nil {
log.Errorf("Failed to mkdir /etc/ceph: %s", err)
Expand All @@ -133,6 +205,29 @@ func InitQemuDeployManager(cpuArch string, hugepage bool, hugepageSizeKB int, de
hugepageSizeKB: hugepageSizeKB,
portsInUse: new(sync.Map),
c: make(chan struct{}, deployConcurrent),
qemuCmd: qemuCmd,
}
}

if manager.runOnHost() {
files, err := ioutil.ReadDir(RUN_ON_HOST_ROOT_PATH)
if err != nil {
return errors.Wrap(err, "readDir RUN_ON_HOST_ROOT_PATH")
}
for _, file := range files {
if file.IsDir() {
continue
}
if strings.HasPrefix(file.Name(), "hmp_") && strings.HasSuffix(file.Name(), ".socket") {
hmpPath := path.Join(RUN_ON_HOST_ROOT_PATH, file.Name())
tryCleanGuest(hmpPath)
}
}

err = procutils.NewCommand("cp", "-rf", "/yunionos", RUN_ON_HOST_ROOT_PATH).Run()
if err != nil {
log.Errorf("Failed to mkdir /opt/cloud/host-deployer: %s", err)
return errors.Wrap(err, "Failed to mkdir /opt/cloud/host-deployer: %s")
}
}

Expand Down Expand Up @@ -431,7 +526,6 @@ type QemuBaseDriver struct {
hugepagePath string
pidPath string
cleaned uint32
sync.Once
}

func (d *QemuBaseDriver) CleanGuest() {
Expand Down Expand Up @@ -470,9 +564,9 @@ type QemuX86Driver struct {

func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error {
uuid := stringutils.UUID4()
socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid)
socketPath := fmt.Sprintf("/opt/cloud/host-deployer/hmp_%s.socket", uuid)

cmd := QEMU_KVM_PATH
cmd := manager.qemuCmd
if sysutils.IsKvmSupport() {
cmd += __("-enable-kvm")
cmd += __("-cpu host")
Expand All @@ -481,7 +575,7 @@ func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
}
cmd += __("-M pc")

d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid)
d.pidPath = fmt.Sprintf("/opt/cloud/host-deployer/%s.pid", uuid)
cmd += __("-nodefaults")
cmd += __("-daemonize")
cmd += __("-monitor unix:%s,server,nowait", socketPath)
Expand Down Expand Up @@ -514,8 +608,8 @@ func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
cmd += __("-m %dM", memSizeMB)
//}

cmd += __("-initrd %s", X86_INITRD_PATH)
cmd += __("-kernel %s", X86_KERNEL_PATH)
cmd += __("-initrd %s", manager.GetX86InitrdPath())
cmd += __("-kernel %s", manager.GetX86KernelPath())
cmd += __("-device VGA")
cmd += __("-device virtio-serial-pci")
cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort)
Expand All @@ -529,7 +623,14 @@ func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
cmd += __("-device ide-cd,drive=ide0-cd0,bus=ide.1")

log.Infof("start guest %s", cmd)
out, err := procutils.NewCommand("bash", "-c", cmd).Output()

var out []byte
var err error
if manager.runOnHost() {
out, err = procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output()
} else {
out, err = procutils.NewCommand("bash", "-c", cmd).Output()
}
if err != nil {
log.Errorf("failed start guest %s: %s", out, err)
return errors.Wrapf(err, "failed start guest %s", out)
Expand Down Expand Up @@ -564,9 +665,9 @@ type QemuARMDriver struct {

func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error {
uuid := stringutils.UUID4()
socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid)
socketPath := fmt.Sprintf("/opt/cloud/host-deployer/hmp_%s.socket", uuid)

cmd := QEMU_KVM_PATH
cmd := manager.qemuCmd
if sysutils.IsKvmSupport() {
cmd += __("-enable-kvm")
cmd += __("-cpu host")
Expand All @@ -575,7 +676,7 @@ func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
}
cmd += __("-M virt,gic-version=max")

d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid)
d.pidPath = fmt.Sprintf("/opt/cloud/host-deployer/%s.pid", uuid)
cmd += __("-nodefaults")
cmd += __("-daemonize")
cmd += __("-monitor unix:%s,server,nowait", socketPath)
Expand All @@ -584,9 +685,13 @@ func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
cmd += __("-smp %d", ncpu)

cmd += __("-m %dM", memSizeMB)
cmd += __("-initrd %s", ARM_INITRD_PATH)
cmd += __("-kernel %s", ARM_KERNEL_PATH)
cmd += __("-drive if=pflash,format=raw,unit=0,file=/usr/share/AAVMF/AAVMF_CODE.fd,readonly=on")
cmd += __("-initrd %s", manager.GetARMInitrdPath())
cmd += __("-kernel %s", manager.GetARMKernelPath())
if manager.runOnHost() {
cmd += __("-drive if=pflash,format=raw,unit=0,file=/opt/cloud/contrib/OVMF.fd,readonly=on")
} else {
cmd += __("-drive if=pflash,format=raw,unit=0,file=/usr/share/AAVMF/AAVMF_CODE.fd,readonly=on")
}

cmd += __("-device virtio-serial-pci")
cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort)
Expand All @@ -600,7 +705,14 @@ func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool,
cmd += __("-device scsi-cd,drive=cd0,share-rw=true")

log.Infof("start guest %s", cmd)
out, err := procutils.NewCommand("bash", "-c", cmd).Output()

var out []byte
var err error
if manager.runOnHost() {
out, err = procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", cmd).Output()
} else {
out, err = procutils.NewCommand("bash", "-c", cmd).Output()
}
if err != nil {
log.Errorf("failed start guest %s: %s", out, err)
return errors.Wrapf(err, "failed start guest %s", out)
Expand Down
21 changes: 14 additions & 7 deletions pkg/hostman/hostdeployer/deployserver/deployserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,11 @@ func (s *SDeployService) PrepareEnv() error {
return errors.Wrapf(err, "cp files failed %s", out)
}

err = procutils.NewCommand("mkdir", "-p", qemu_kvm.RUN_ON_HOST_ROOT_PATH).Run()
if err != nil {
return errors.Wrap(err, "Failed to mkdir RUN_ON_HOST_ROOT_PATH: %s")
}

cmd := fmt.Sprintf("mkisofs -l -J -L -R -r -v -hide-rr-moved -o %s -graft-points vmware-vddk=/opt/vmware-vddk yunion=/opt/yunion", qemu_kvm.DEPLOY_ISO)
out, err = procutils.NewCommand("bash", "-c", cmd).Output()
if err != nil {
Expand All @@ -371,8 +376,10 @@ func (s *SDeployService) PrepareEnv() error {
if err != nil {
return errors.Wrap(err, "get cpu architecture")
}

err = qemu_kvm.InitQemuDeployManager(
strings.TrimSpace(string(cpuArch)),
DeployOption.DefaultQemuVersion,
DeployOption.HugepagesOption == "native",
DeployOption.HugepageSizeMb*1024,
DeployOption.DeployConcurrent,
Expand Down Expand Up @@ -430,20 +437,20 @@ func (s *SDeployService) InitService() {
}
})

if DeployOption.EnableRemoteExecutor {
log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath)
execlient.Init(DeployOption.ExecutorSocketPath)
execlient.SetTimeoutSeconds(DeployOption.ExecutorConnectTimeoutSeconds)
procutils.SetRemoteExecutor()
}

if DeployOption.DeployAction != "" {
if err := LocalInitEnv(); err != nil {
log.Fatalf("local init env %s", err)
}
return
}

log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath)
if DeployOption.EnableRemoteExecutor {
execlient.Init(DeployOption.ExecutorSocketPath)
execlient.SetTimeoutSeconds(DeployOption.ExecutorConnectTimeoutSeconds)
procutils.SetRemoteExecutor()
}

if err := s.PrepareEnv(); err != nil {
log.Fatalln(err)
}
Expand Down
9 changes: 6 additions & 3 deletions pkg/hostman/hostdeployer/deployserver/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@ type SDeployOptions struct {

AllowVmSELinux bool `help:"turn off vm selinux" default:"false" json:"allow_vm_selinux"`

HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"`
HugepageSizeMb int `help:"hugepage size mb default 1G" default:"1024"`
HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"`
HugepageSizeMb int `help:"hugepage size mb default 1G" default:"1024"`
DefaultQemuVersion string `help:"Default qemu version" default:"4.2.0"`
ListenInterface string `help:"Master address of host server"`
Networks []string `help:"Network interface information"`

DeployAction string `help:"local deploy action"`
DeployParams string `help:"params for deploy action"`
Expand All @@ -44,7 +47,7 @@ type SDeployOptions struct {
var DeployOption SDeployOptions

func Parse() (hostOpts SDeployOptions) {
common_options.ParseOptions(&hostOpts, os.Args, "host.conf", "host")
common_options.ParseOptionsIgnoreNoConfigfile(&hostOpts, os.Args, "host.conf", "host")
if len(hostOpts.CommonConfigFile) > 0 {
commonCfg := &common_options.HostCommonOptions{}
commonCfg.Config = hostOpts.CommonConfigFile
Expand Down

0 comments on commit 9052d5e

Please sign in to comment.