Skip to content

Commit

Permalink
Addressed review comments
Browse files Browse the repository at this point in the history 
Signed-off-by: Jaroslaw Kurowski <[email protected]>
  • Loading branch information
@jkuro-tii
jkuro-tii committed Oct 23, 2024
1 parent b42c14f commit 2ac570d
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 142 deletions.
291 changes: 153 additions & 138 deletions modules/hardware/common/shared-mem.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,22 @@
pkgs,
...
}:
with lib;
let
cfg = config.ghaf.shm;
inherit (lib)
foldl'
lists
mkMerge
mkIf
mkOption
mdDoc
types
;
in
{
options.ghaf.shm = {
enable = lib.mkOption {
type = lib.types.bool;
enable = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enables using shared memory between VMs and the host.
Expand All @@ -26,6 +37,20 @@ with lib;
Defines shared memory size in MBytes.
'';
};
hugePageSz = mkOption {
type = types.str;
default = "2M";
description = mdDoc ''
Defines big memory area page size. Kernel supported values are
2M and 1G.
'';
apply =
value:
if value != "2M" && value != "1G" then
builtins.throw "Invalid huge memory area page size"
else
value;
};
hostSocketPath = mkOption {
type = types.path;
default = "/tmp/ivshmem_socket"; # The value is hardcoded in the application
Expand All @@ -52,26 +77,17 @@ with lib;
specific, in order not to conflict with other memory areas (e.g. PCI).
'';
};
enable_host = lib.mkOption {
type = lib.types.bool;
enable_host = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enables memsocket on host.
'';
apply =
enable:
if enable && !config.ghaf.shm.enable then
builtins.throw "Enable shared memory in order to use shm on host"
else
enable;
};
instancesCount = mkOption {
type = types.int;
default =
if config.ghaf.shm.enable_host then
(builtins.length config.ghaf.shm.vms_enabled) + 1
else
builtins.length config.ghaf.shm.vms_enabled;
if cfg.enable_host then (builtins.length cfg.vms_enabled) + 1 else builtins.length cfg.vms_enabled;
description = mdDoc ''
Number of VMs entitled to use shared memory.
'';
Expand All @@ -97,136 +113,135 @@ with lib;
type = types.bool;
default = false;
description = "Display VMs using shared memory";
apply =
enable:
if enable && !config.ghaf.shm.enable then
builtins.throw "Enable shared memory in order to use shm display"
else
enable;
};
};

config.boot.kernelParams =
let
hugepagesz = "2M"; # valid values: "2M" and "1G", as kernel supports these huge pages' size
hugepages =
if hugepagesz == "2M" then config.ghaf.shm.memSize / 2 else config.ghaf.shm.memSize / 1024;
in
optionals config.ghaf.shm.enable [
"hugepagesz=${hugepagesz}"
"hugepages=${toString hugepages}"
];
config.environment.systemPackages = optionals config.ghaf.shm.enable_host [
(pkgs.callPackage ../../../packages/memsocket { vms = config.ghaf.shm.instancesCount; })
];

config.systemd.services.ivshmemsrv =
let
pidFilePath = "/tmp/ivshmem-server.pid";
ivShMemSrv =
config = mkIf cfg.enable (mkMerge [
{
boot.kernelParams =
let
vectors = toString (2 * config.ghaf.shm.instancesCount);
hugepages = if cfg.hugePageSz == "2M" then cfg.memSize / 2 else cfg.memSize / 1024;
in
pkgs.writeShellScriptBin "ivshmemsrv" ''
chown microvm /dev/hugepages
chgrp kvm /dev/hugepages
if [ -S ${config.ghaf.shm.hostSocketPath} ]; then
echo Erasing ${config.ghaf.shm.hostSocketPath} ${pidFilePath}
rm -f ${config.ghaf.shm.hostSocketPath}
fi
${pkgs.sudo}/sbin/sudo -u microvm -g kvm ${pkgs.qemu_kvm}/bin/ivshmem-server -p ${pidFilePath} -n ${vectors} -m /dev/hugepages/ -l ${
(toString config.ghaf.shm.memSize) + "M"
}
'';
in
lib.mkIf config.ghaf.shm.enable {
enable = true;
description = "Start qemu ivshmem memory server";
path = [ ivShMemSrv ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StandardOutput = "journal";
StandardError = "journal";
ExecStart = "${ivShMemSrv}/bin/ivshmemsrv";
};
};
config.microvm.vms =
let
memsocket = pkgs.callPackage ../../../packages/memsocket { vms = config.ghaf.shm.instancesCount; };
vectors = toString (2 * config.ghaf.shm.instancesCount);
makeAssignment = vmName: {
${vmName} = {
config = {
config = {
microvm = {
qemu = {
extraArgs = [
"-device"
"ivshmem-doorbell,vectors=${vectors},chardev=ivs_socket,flataddr=${config.ghaf.shm.flataddr}"
"-chardev"
"socket,path=${config.ghaf.shm.hostSocketPath},id=ivs_socket"
];
};
kernelParams = [ "kvm_ivshmem.flataddr=${config.ghaf.shm.flataddr}" ];
};
boot.extraModulePackages = [
(pkgs.linuxPackages.callPackage ../../../packages/memsocket/module.nix {
inherit (config.microvm.vms.${vmName}.config.config.boot.kernelPackages) kernel;
vmCount = config.ghaf.shm.instancesCount;
})
];
services = {
udev = {
extraRules = ''
SUBSYSTEM=="misc",KERNEL=="ivshmem",GROUP="kvm",MODE="0666"
'';
};
};
environment.systemPackages = optionals config.ghaf.shm.enable [
memsocket
];
systemd.user.services.memsocket =
if vmName == "gui-vm" then
lib.mkIf config.ghaf.shm.enable {
enable = true;
description = "memsocket";
after = [ "labwc.service" ];
serviceConfig = {
Type = "simple";
ExecStart = "${memsocket}/bin/memsocket -c ${config.ghaf.shm.clientSocketPath}";
Restart = "always";
RestartSec = "1";
[
"hugepagesz=${cfg.hugePageSz}"
"hugepages=${toString hugepages}"
];
}
(mkIf cfg.enable_host {
environment.systemPackages = [
(pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; })
];
})
{
systemd.services.ivshmemsrv =
let
user = "microvm";
group = "kvm";
pidFilePath = "/tmp/ivshmem-server.pid";
ivShMemSrv =
let
vectors = toString (2 * cfg.instancesCount);
in
pkgs.writeShellScriptBin "ivshmemsrv" ''
chown ${user} /dev/hugepages
chgrp ${group} /dev/hugepages
if [ -S ${cfg.hostSocketPath} ]; then
echo Erasing ${cfg.hostSocketPath} ${pidFilePath}
rm -f ${cfg.hostSocketPath}
fi
${pkgs.sudo}/sbin/sudo -u ${user} -g ${group} ${pkgs.qemu_kvm}/bin/ivshmem-server -p ${pidFilePath} -n ${vectors} -m /dev/hugepages/ -l ${(toString cfg.memSize) + "M"}
'';
in
{
enable = true;
description = "Start qemu ivshmem memory server";
path = [ ivShMemSrv ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StandardOutput = "journal";
StandardError = "journal";
ExecStart = "${ivShMemSrv}/bin/ivshmemsrv";
};
};
}
{
microvm.vms =
let
memsocket = pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; };
vectors = toString (2 * cfg.instancesCount);
makeAssignment = vmName: {
${vmName} = {
config = {
config = {
microvm = {
qemu = {
extraArgs = [
"-device"
"ivshmem-doorbell,vectors=${vectors},chardev=ivs_socket,flataddr=${cfg.flataddr}"
"-chardev"
"socket,path=${cfg.hostSocketPath},id=ivs_socket"
];
};
wantedBy = [ "ghaf-session.target" ];
}
else
# machines connecting to gui-vm
let
vmIndex = lib.lists.findFirstIndex (vm: vm == vmName) null config.ghaf.shm.vms_enabled;
in
lib.mkIf config.ghaf.shm.enable {
enable = true;
description = "memsocket";
serviceConfig = {
Type = "simple";
ExecStart = "${memsocket}/bin/memsocket -s ${config.ghaf.shm.serverSocketPath} ${builtins.toString vmIndex}";
Restart = "always";
RestartSec = "1";
kernelParams = [ "kvm_ivshmem.flataddr=${cfg.flataddr}" ];
};
boot.extraModulePackages = [
(pkgs.linuxPackages.callPackage ../../../packages/memsocket/module.nix {
inherit (config.microvm.vms.${vmName}.config.config.boot.kernelPackages) kernel;
vmCount = cfg.instancesCount;
})
];
services = {
udev = {
extraRules = ''
SUBSYSTEM=="misc",KERNEL=="ivshmem",GROUP="kvm",MODE="0666"
'';
};
wantedBy = [ "default.target" ];
};
environment.systemPackages = [
memsocket
];
systemd.user.services.memsocket =
if vmName == "gui-vm" then
{
enable = true;
description = "memsocket";
after = [ "labwc.service" ];
serviceConfig = {
Type = "simple";
ExecStart = "${memsocket}/bin/memsocket -c ${cfg.clientSocketPath}";
Restart = "always";
RestartSec = "1";
};
wantedBy = [ "ghaf-session.target" ];
}
else
# machines connecting to gui-vm
let
vmIndex = lists.findFirstIndex (vm: vm == vmName) null cfg.vms_enabled;
in
{
enable = true;
description = "memsocket";
serviceConfig = {
Type = "simple";
ExecStart = "${memsocket}/bin/memsocket -s ${cfg.serverSocketPath} ${builtins.toString vmIndex}";
Restart = "always";
RestartSec = "1";
};
wantedBy = [ "default.target" ];
};
};
};
};
};
};
};
in
mkIf config.ghaf.shm.enable (
foldl' lib.attrsets.recursiveUpdate { } (map makeAssignment config.ghaf.shm.vms_enabled)
);

config.ghaf.hardware.definition.gpu.kernelConfig.kernelParams = optionals config.ghaf.shm.enable [
"kvm_ivshmem.flataddr=${config.ghaf.shm.flataddr}"
];
in
foldl' lib.attrsets.recursiveUpdate { } (map makeAssignment cfg.vms_enabled);
}
{
microvm.vms.gui-vm.config.config.boot.kernelParams = [
"kvm_ivshmem.flataddr=${cfg.flataddr}"
];
}
]);
}
4 changes: 2 additions & 2 deletions packages/memsocket/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ stdenv.mkDerivation {
src = fetchFromGitHub {
owner = "tiiuae";
repo = "shmsockproxy";
rev = "fd6b8ac051f1c68edc2b3222146d5d9053c81cc6";
sha256 = "sha256-4FiRZy/HxwANgIAHxUfQo2tl4Dcgpz74SqvuRjqIw8M=";
rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1";
sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s=";
};

CFLAGS = "-O2 -DVM_COUNT=" + (toString vms) + (if debug then " -DDEBUG_ON" else "");
Expand Down
4 changes: 2 additions & 2 deletions packages/memsocket/module.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ stdenv.mkDerivation {
src = fetchFromGitHub {
owner = "tiiuae";
repo = "shmsockproxy";
rev = "851ca1f2d23db764dd817b15aa783d82ab17560f";
sha256 = "sha256-8jyciVZccptGaj4u3bDj5YOCfZSsf69FH4yfqcUoB5k=";
rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1";
sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s=";
};

sourceRoot = "source/module";
Expand Down

0 comments on commit 2ac570d

Please sign in to comment.