Memory sharing for wayland displaying

Signed-off-by: Jaroslaw Kurowski <[email protected]>

modules/development/debug-tools.nix

@@ -37,6 +37,9 @@ in
           # Performance testing
           speedtest-cli
           iperf
+          perf-tools
+          linuxPackages.perf
+
         ]
         ++
         # LuaJIT (which is sysbench dependency) not available on RISC-V

modules/profiles/applications.nix

@@ -13,6 +13,47 @@ in
       enable = mkEnableOption "Some sample applications";
       #TODO Create options to allow enabling individual apps
       #weston.ini.nix mods needed
+      ivShMemServer = {
+        memSize = mkOption {
+          type = lib.types.str;
+          default = "16M";
+          description = mdDoc ''
+          Defines shared memory size
+        '';
+        };
+        vmCount = mkOption {
+          type = lib.types.int;
+          default = 6;
+          description = mdDoc ''
+          Defines maximum number of application VMs
+        '';
+        };
+        serverSocketPath = mkOption {
+          type = lib.types.str;
+          default = "/tmp/memsocket-server.sock";
+          description = mdDoc ''
+          Defines location of the listening socket.
+          It's used by waypipe as an output socket when running in server mode
+        '';
+        };
+        clientSocketPath = mkOption {
+          type = lib.types.str;
+          default = "/tmp/memsocket-client.sock";
+          description = mdDoc ''
+          Defines location of the output socket. It's outputed
+          with data coming from AppVMs.
+          It's used by waypipe as an input socket when running in client mode
+        '';
+        };
+          hostSocketPath = mkOption {
+          type = lib.types.str;
+          default = "/tmp/ivshmem_socket";
+          description = mdDoc ''
+          Defines location of the shared memory socket. It's used by qemu 
+          instances for memory sharing and sending interrupts.
+        '';
+        };
+      };
     };
 
     config = mkIf cfg.enable {

modules/virtualization/microvm/appvm.nix

@@ -3,6 +3,7 @@
 {
   config,
   lib,
+  pkgs,
   ...
 }: let
   configHost = config;
@@ -16,6 +17,9 @@
       if vm.cid > 0
       then vm.cid
       else cfg.vsockBaseCID + index;
+    memsocket = pkgs.callPackage ../../../user-apps/memsocket {
+      debug = false; vms = config.ghaf.profiles.applications.ivShMemServer.vmCount;
+    };
     appvmConfiguration = {
       imports = [
         (import ./common/vm-networking.nix {
@@ -65,6 +69,8 @@
 
           environment.systemPackages = [
             pkgs.waypipe
+            memsocket
+            pkgs.perf-tools pkgs.linuxPackages.perf
           ];
 
           microvm = {
@@ -89,10 +95,25 @@
             qemu.extraArgs = [
               "-M"
               "q35,accel=kvm:tcg,mem-merge=on,sata=off"
-              "-device"
-              "vhost-vsock-pci,guest-cid=${toString cid}"
             ];
           };
+
+          services.udev.extraRules = ''
+            SUBSYSTEM=="misc",KERNEL=="ivshmem",GROUP="kvm",MODE="0666"
+          '';
+
+          systemd.user.services.memsocket = {
+            enable = true;
+            description = "memsocket";
+            serviceConfig = {
+              Type = "simple";
+              ExecStart = "${memsocket}/bin/memsocket -s ${config.ghaf.profiles.applications.ivShMemServer.serverSocketPath} ${builtins.toString cid}";
+              Restart = "always";
+              RestartSec = "1";
+            };
+            wantedBy = ["default.target"];
+          };
+
           fileSystems."/run/waypipe-ssh-public-key".options = ["ro"];
 
           imports = import ../../module-list.nix;
@@ -101,7 +122,19 @@
     };
   in {
     autostart = true;
-    config = appvmConfiguration // {imports = appvmConfiguration.imports ++ cfg.extraModules ++ vm.extraModules ++ [{environment.systemPackages = vm.packages;}];};
+    config = appvmConfiguration // {imports = appvmConfiguration.imports ++ cfg.extraModules ++ vm.extraModules ++ [{environment.systemPackages = vm.packages;}];
+    } // {
+      boot.kernelPatches = [{
+        name = "Shared memory PCI driver";
+        patch = pkgs.fetchpatch {
+          url = "https://raw.githubusercontent.com/tiiuae/shmsockproxy/dev/0001-ivshmem-driver.patch";
+          sha256 = "sha256-SBeVxHqoyZNa7Q4iLfJbppqHQyygKGRJjtJGHh04DZA=";
+        };
+        extraConfig = ''
+          KVM_IVSHMEM_VM_COUNT ${toString config.ghaf.profiles.applications.ivShMemServer.vmCount}
+          '';
+      }];
+    };
     specialArgs = {inherit lib;};
   };
 in {
@@ -174,11 +207,11 @@ in {
     };
 
     # Base VSOCK CID which is used for auto assigning CIDs for all AppVMs
-    # For example, when it's set to 100, AppVMs will get 100, 101, 102, etc.
+    # For example, when it's set to 0, AppVMs will get 0, 1, 2, etc.
     # It is also possible to override the auto assinged CID using the vms.cid option
     vsockBaseCID = lib.mkOption {
       type = lib.types.int;
-      default = 100;
+      default = 0;
       description = ''
         Context Identifier (CID) of the AppVM VSOCK
       '';

modules/virtualization/microvm/guivm.nix

@@ -9,6 +9,9 @@
   configHost = config;
   vmName = "gui-vm";
   macAddress = "02:00:00:02:02:02";
+  memsocket = pkgs.callPackage ../../../user-apps/memsocket {
+    debug = false; vms = config.ghaf.profiles.applications.ivShMemServer.vmCount;
+  };
   guivmBaseConfiguration = {
     imports = [
       (import ./common/vm-networking.nix {inherit vmName macAddress;})
@@ -52,11 +55,15 @@
           };
         };
 
-        environment = {
+        environment = with pkgs; {
           systemPackages = [
             pkgs.waypipe
             pkgs.networkmanagerapplet
             pkgs.nm-launcher
+            memsocket
+            # debugging
+            pkgs.perf-tools
+            linuxPackages.perf
           ];
         };
 
@@ -84,29 +91,57 @@
           ];
           writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";
 
-          qemu.extraArgs = [
+          qemu.extraArgs =
+            let vectors = (toString (2 * config.ghaf.profiles.applications.ivShMemServer.vmCount)); in [
+            "-object"
+            "memory-backend-file,size=${config.ghaf.profiles.applications.ivShMemServer.memSize},share=on,mem-path=/dev/shm/ivshmem,id=hostmem"
             "-device"
-            "vhost-vsock-pci,guest-cid=${toString cfg.vsockCID}"
+            "ivshmem-doorbell,vectors=${vectors},chardev=ivs_socket"
+            "-chardev"
+            "socket,path=${config.ghaf.profiles.applications.ivShMemServer.hostSocketPath},id=ivs_socket"
           ];
         };
 
         imports = import ../../module-list.nix;
 
+        services.udev.extraRules = ''
+          SUBSYSTEM=="misc",KERNEL=="ivshmem",GROUP="kvm",MODE="0666"
+          '';
+
         # Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs
-        systemd.user.services.waypipe = {
-          enable = true;
-          description = "waypipe";
-          after = ["weston.service" "labwc.service"];
-          serviceConfig = {
-            Type = "simple";
-            ExecStart = "${pkgs.waypipe}/bin/waypipe --vsock -s ${toString cfg.waypipePort} client";
-            Restart = "always";
-            RestartSec = "1";
+        # via shared memory socket
+        systemd.user.services = {
+          waypipe = {
+            enable = true;
+            description = "waypipe";
+            after = ["weston.service" "labwc.service" "memsocket.service"];
+            serviceConfig = {
+              Type = "simple";
+              ExecStart = "${pkgs.waypipe}/bin/waypipe -s ${config.ghaf.profiles.applications.ivShMemServer.clientSocketPath} client";
+              Restart = "always";
+              RestartSec = "1";
+            };
+            startLimitIntervalSec = 0;
+            wantedBy = ["ghaf-session.target"];
           };
-          startLimitIntervalSec = 0;
-          wantedBy = ["ghaf-session.target"];
-        };
 
+          # Waypipe in GUIVM needs to communicate with AppVMs using socket forwading
+          # application. It uses shared memory between virtual machines to forward
+          # data between sockets.
+          #
+          memsocket = {
+            enable = true;
+            description = "memsocket";
+            after = ["weston.service"];
+            serviceConfig = {
+              Type = "simple";
+              ExecStart = "${memsocket}/bin/memsocket -c ${config.ghaf.profiles.applications.ivShMemServer.clientSocketPath}";
+              Restart = "always";
+              RestartSec = "1";
+            };
+            wantedBy = ["ghaf-session.target"];
+          };
+        };
         # Fixed IP-address for debugging subnet
         systemd.network.networks."10-ethint0".addresses = [
           {
@@ -117,40 +152,17 @@
     ];
   };
   cfg = config.ghaf.virtualization.microvm.guivm;
-  vsockproxy = pkgs.callPackage ../../../packages/vsockproxy {};
-in {
-  options.ghaf.virtualization.microvm.guivm = {
-    enable = lib.mkEnableOption "GUIVM";
-
-    extraModules = lib.mkOption {
-      description = ''
-        List of additional modules to be imported and evaluated as part of
-        GUIVM's NixOS configuration.
-      '';
-      default = [];
-    };
-
-    # GUIVM uses a VSOCK which requires a CID
-    # There are several special addresses:
-    # VMADDR_CID_HYPERVISOR (0) is reserved for services built into the hypervisor
-    # VMADDR_CID_LOCAL (1) is the well-known address for local communication (loopback)
-    # VMADDR_CID_HOST (2) is the well-known address of the host
-    # CID 3 is the lowest available number for guest virtual machines
-    vsockCID = lib.mkOption {
-      type = lib.types.int;
-      default = 3;
-      description = ''
-        Context Identifier (CID) of the GUIVM VSOCK
-      '';
-    };
+  in {
+    options.ghaf.virtualization.microvm.guivm = {
+      enable = lib.mkEnableOption "GUIVM";
 
-    waypipePort = lib.mkOption {
-      type = lib.types.int;
-      default = 1100;
-      description = ''
-        Waypipe port number to listen for incoming connections from AppVMs
-      '';
-    };
+      extraModules = lib.mkOption {
+        description = ''
+          List of additional modules to be imported and evaluated as part of
+          GUIVM's NixOS configuration.
+        '';
+        default = [];
+      };
   };
 
   config = lib.mkIf cfg.enable {
@@ -162,7 +174,17 @@ in {
           imports =
             guivmBaseConfiguration.imports
             ++ cfg.extraModules;
-        };
+        } // {
+          config.boot.kernelPatches = [{
+            name = "Shared memory PCI driver";
+            patch = pkgs.fetchpatch {
+              url = "https://raw.githubusercontent.com/tiiuae/shmsockproxy/dev/0001-ivshmem-driver.patch";
+              sha256 = "sha256-SBeVxHqoyZNa7Q4iLfJbppqHQyygKGRJjtJGHh04DZA=";
+            };
+            extraConfig = ''
+              KVM_IVSHMEM_VM_COUNT ${toString config.ghaf.profiles.applications.ivShMemServer.vmCount}
+            '';
+          }];};
       specialArgs = {inherit lib;};
     };
 
@@ -186,20 +208,30 @@ in {
       };
     };
 
-    # Waypipe in GUIVM needs to communicate with AppVMs over VSOCK
-    # However, VSOCK does not support direct guest to guest communication
-    # The vsockproxy app is used on host as a bridge between AppVMs and GUIVM
-    # It listens for incoming connections from AppVMs and forwards data to GUIVM
-    systemd.services.vsockproxy = {
+    systemd.services.ivshmemsrv = let
+      socketPath = config.ghaf.profiles.applications.ivShMemServer.hostSocketPath;
+      pidFilePath = "/tmp/ivshmem-server.pid";
+      ivShMemSrv =
+          let vectors = (toString (2 * config.ghaf.profiles.applications.ivShMemServer.vmCount)); in
+        pkgs.writeShellScriptBin "ivshmemsrv" ''
+          if [ -S ${socketPath} ]; then
+            echo Erasing ${socketPath} ${pidFilePath}
+            rm -f ${socketPath}
+          fi
+          ${pkgs.sudo}/sbin/sudo -u microvm -g kvm ${pkgs.qemu_kvm}/bin/ivshmem-server -p ${pidFilePath} -n ${vectors} -m /dev/shm -l ${config.ghaf.profiles.applications.ivShMemServer.memSize}
+        '';
+    in {
       enable = true;
-      description = "vsockproxy";
-      unitConfig = {
-        Type = "simple";
-      };
+      description = "Start qemu ivshmem memory server";
+      path = [ivShMemSrv];
+      wantedBy = ["multi-user.target"];
       serviceConfig = {
-        ExecStart = "${vsockproxy}/bin/vsockproxy ${toString cfg.waypipePort} ${toString cfg.vsockCID} ${toString cfg.waypipePort}";
+        Type = "oneshot";
+        RemainAfterExit = true;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        ExecStart = "${ivShMemSrv}/bin/ivshmemsrv";
       };
-      wantedBy = ["multi-user.target"];
     };
   };
 }

overlays/custom-packages/qemu/default.nix

@@ -14,5 +14,10 @@
         // (final.lib.optionalAttrs (qemu_major == "8" && qemu_minor == "1") {
           patches = prev.patches ++ [./acpi-devices-passthrough-qemu-8.1.patch];
         })
+        // {
+          postInstall = (prev.postInstall or "") + ''
+            cp contrib/ivshmem-server/ivshmem-server $out/bin
+          '';
+        }
     );
 })