From 4799a6ffe5fc81c92d5e16042af4787942b39f12 Mon Sep 17 00:00:00 2001 From: Jon Sahlberg Date: Mon, 4 Dec 2023 12:03:48 +0200 Subject: [PATCH] architecture: Initial Audio VM adr Initial documentation for Audio virtual machine to separate the audio data away from host. Audio stream is routed through host over a virtual sound card to separate the virtual machines from eachothers audio streams. Signed-off-by: Jon Sahlberg Signed-off-by: Ville Ilvonen --- docs/src/architecture/adr.md | 1 + docs/src/architecture/adr/audiovm.md | 94 +++++++++++++++++ modules/hardware/definition.nix | 6 +- modules/host/networking.nix | 3 + modules/virtualization/microvm/audiovm.nix | 42 +++++--- modules/virtualization/microvm/guivm.nix | 2 + overlays/custom-packages/tcp-ssh/default.nix | 4 +- {user-apps => packages}/tcp-ssh/default.nix | 8 +- targets/lenovo-x1-carbon.nix | 100 +++++++++++++++++-- 9 files changed, 227 insertions(+), 33 deletions(-) create mode 100644 docs/src/architecture/adr/audiovm.md rename {user-apps => packages}/tcp-ssh/default.nix (91%) diff --git a/docs/src/architecture/adr.md b/docs/src/architecture/adr.md index cfd7624426..7c0a72bd9f 100644 --- a/docs/src/architecture/adr.md +++ b/docs/src/architecture/adr.md @@ -14,6 +14,7 @@ The Ghaf platform decision log: | [Minimal Host](../architecture/adr/minimal-host.md) | Proposed. | | [netvm—Networking Virtual Machine](../architecture/adr/netvm.md) | Proposed, partially implemented for development and testing. | | [Platform Bus for RustVMM](../architecture/adr/platform-bus-passthrough-support.md) | Proposed, WIP. | +| [audiovm—Audio Virtual Machine ](../architecture/adr/audiovm.md) | Proposed, WIP. | To create an architectural decision proposal, open [a pull request](https://github.com/tiiuae/ghaf/blob/main/CONTRIBUTING.md#contributing-documentation) and use the [decision record template](https://github.com/tiiuae/ghaf/blob/main/docs/src/architecture/adr/template.md). Contributions to the Ghaf architecture decisions are welcome. diff --git a/docs/src/architecture/adr/audiovm.md b/docs/src/architecture/adr/audiovm.md new file mode 100644 index 0000000000..9da738445a --- /dev/null +++ b/docs/src/architecture/adr/audiovm.md @@ -0,0 +1,94 @@ + + +# audiovm-Audio support Virtual Machine + +## Status + +Proposed, partially implemented for development and testing. + +*audiovm* reference declaration will be available at [microvm/audiovm.nix] +(https://github.com/tiiuae/ghaf/blob/main/modules/virtualization/microvm/audiovm.nix) + +## Context + +Ghaf high-level design target is to secure a monolithic OS by modularizing +the OS. The key security target is to limit user data leakage between OS, +applications and components. This isolates audio data between virtual +machines and allows better control over which virtual machine gets access +to audio peripherals. + +## Decision + +The two main goals are to isolate the virtual machine audio streams to prevent +audio data leakage between virtual machines and allow separate control over +which virtual machine is allowed to use audio peripherals at given time. The +other goal is to isolate the audio stream processing to a separate virtual +machine from the host to reduce size of trusted computing base in the host. + +Separating the audio processing away from the host is needed because some +guest virtual machines need direct access to the main audio stream controls +and allowing a direct connection from guests to host services would not be +ideal for security perspective. + +On the host, the audio device is passed through to a dedicated Audio virtual +machine (VM). AudioVM provides a service to let other virtual machines to use +the audio peripherals. Letting the application virtual machines to connect to +the audioVM backend directly could allow the application virtual machies to +have full controll of all audio data including audio of other virtual +machines. For this reason the application virtual machine audio is routed +through the virtual machine manager. Each application virtual machine gets +its own virtualized sound card device from the virtual machine manager. +The virtual machine manager routes the Application virtual machines virtual +sound card data to actual sound service on AudioVM. + + ┌─────────────────┐ ┌──────────────────┐ + │ │ │ │ + │ │ │ │ + Direct Audio │ GuiVM │ │ AppVM2 │ + ┌──────────┤ │ │ │ + │ │ │ ┌──────►│ │ + │ └─────────────────┘ │ └──────────────────┘ + │ │ Virtual Audio Device + │ │ + │ │ + ┌──────▼──────────┐ │ ┌──────────────────┐ + │ │ │ │ │ + │ │ VM Audio │ │ │ + │ AudioVM ◄─────────────┐ │ │ AppVM1 │ + │ │ │ │ │ │ + │ │ │ │ │ │ + └──────────────▲──┘ │ │ └─────────▲────────┘ + │ ┌───┴─────────────┴─┐ │ + │ │ │ │ + Audio device │ │ VM Manager ├───────────────┘ + passthrough │ │ │ Virtual Audio Device + ┌┴────────────┼───────────────────┤ + │ └───────────────────┤ + │ │ + │ Host │ + │ │ + │ │ + └─────────────────────────────────┘ + + +## Consequences + +As with most passthroughs there may be some hardware dependencies which need +to be handeled case by case. With our reference device Lenovo X1 laptop the +integrated sound card is set in the same VFIO group with the ISA controller, +SMBus controller and a Serial bus controller. Passing through a single device +practically forces passing through all the devices in the same VFIO group to +the same virtual machine. In some scenarios that can make some of the critical +system components less isolated and allow one incident to spread to other +components more easily. However, this is the reference hardware design related +consequence. Hardware designs with device and device group connections that +support device isolation - either pass through or paravirtualized - do not +require sharing in the same virtual machine. + +The GUI virtual machine needs the ability to control over all the separate +virtual machine sound channels. This may pose a threat for the Gand possibly +also over the sound data. The level of control over the sound data should be +reviewed as that depends on which sound subsystem is being used on AudioVM. diff --git a/modules/hardware/definition.nix b/modules/hardware/definition.nix index 1e9f0ab444..804d333fbd 100644 --- a/modules/hardware/definition.nix +++ b/modules/hardware/definition.nix @@ -75,9 +75,9 @@ }; audio = { - # TODO? Should add AudioVM enabler here? - # audiovm.enable = mkEnableOption = "AudioVM"; - + # With the current implementation, the whole PCI IOMMU group 14: + # 00:1f.x in the example from Lenovo X1 Carbon + # must be defined for passthrough to AudioVM pciDevices = mkOption { description = "PCI Devices to passthrough to AudioVM"; type = types.listOf pciDevSubmodule; diff --git a/modules/host/networking.nix b/modules/host/networking.nix index 589902b3af..dcb375eac7 100644 --- a/modules/host/networking.nix +++ b/modules/host/networking.nix @@ -18,6 +18,9 @@ in enableIPv6 = false; useNetworkd = true; interfaces.virbr0.useDHCP = false; + extraHosts = '' + 192.168.101.4 audio-vm.lan + ''; }; systemd.network = { diff --git a/modules/virtualization/microvm/audiovm.nix b/modules/virtualization/microvm/audiovm.nix index ea26f16615..e44c3def4d 100644 --- a/modules/virtualization/microvm/audiovm.nix +++ b/modules/virtualization/microvm/audiovm.nix @@ -3,7 +3,6 @@ { config, lib, - pkgs, ... }: let configHost = config; @@ -11,7 +10,10 @@ macAddress = "02:00:00:02:03:04"; audiovmBaseConfiguration = { imports = [ - (import ./common/vm-networking.nix {inherit vmName macAddress;}) + (import ./common/vm-networking.nix { + inherit vmName; + inherit macAddress; + }) ({ lib, pkgs, @@ -43,31 +45,41 @@ sound.enable = true; hardware.pulseaudio.enable = true; hardware.pulseaudio.systemWide = true; -# networking.firewall.interfaces.virbr0.allowedTCPPorts = [4713]; - networking.firewall.allowedTCPPorts = [4713]; + networking.firewall = { + interfaces = { + # Allow tcp 4710 for pulseaudio tcp control (localhost) + # Allow udp 9875 for pulseaudio RTP input (host) + "virbr0" = { + allowedTCPPorts = [4713]; + allowedUDPPorts = [9875]; + }; + }; + }; - # Allow microvm user to access pulseaudio + # Allow ghaf user to access pulseaudio users.extraUsers.ghaf.extraGroups = ["audio" "pulse-access"]; # Enable and allow TCP connection from localhost only hardware.pulseaudio.tcp.enable = true; - hardware.pulseaudio.tcp.anonymousClients.allowedIpRanges = ["127.0.0.1" "192.168.101.2" "192.168.101.3"]; + hardware.pulseaudio.tcp.anonymousClients.allowedIpRanges = ["127.0.0.1"]; hardware.pulseaudio.extraConfig = '' - load-module module-native-protocol-unix - load-module module-native-protocol-tcp auth-ip-acl=127.0.0.1;192.168.101.2;192.168.101.3 - set-sink-volume @DEFAULT_SINK@ 60000 + load-module module-native-protocol-tcp auth-ip-acl=127.0.0.1 + # Listen for UDP connection from host (without broadcast) + load-module module-rtp-recv latency_msec=10 sap_address=0.0.0.0 + set-sink-volume @DEFAULT_SINK@ 60000 ''; + hardware.pulseaudio.daemon.config = { + default-sample-rate = 44100; + alternate-sample-rate = 48000; + avoid-resampling = "yes"; + }; + microvm = { optimize.enable = false; vcpu = 1; - mem = 1024; + mem = 256; hypervisor = "qemu"; shares = [ - { - tag = "rw-waypipe-ssh-public-key"; - source = "/run/waypipe-ssh-public-key"; - mountPoint = "/run/waypipe-ssh-public-key"; - } { tag = "ro-store"; source = "/nix/store"; diff --git a/modules/virtualization/microvm/guivm.nix b/modules/virtualization/microvm/guivm.nix index a2deb1086c..b51499b287 100644 --- a/modules/virtualization/microvm/guivm.nix +++ b/modules/virtualization/microvm/guivm.nix @@ -57,6 +57,8 @@ pkgs.waypipe pkgs.networkmanagerapplet pkgs.nm-launcher + pkgs.pamixer + (pkgs.callPackage ../../../packages/tcp-ssh {}) ]; }; diff --git a/overlays/custom-packages/tcp-ssh/default.nix b/overlays/custom-packages/tcp-ssh/default.nix index b5534e33ae..0c5b7adb70 100644 --- a/overlays/custom-packages/tcp-ssh/default.nix +++ b/overlays/custom-packages/tcp-ssh/default.nix @@ -1,5 +1,5 @@ # Copyright 2022-2023 TII (SSRC) and the Ghaf contributors # SPDX-License-Identifier: Apache-2.0 -(final: prev: { - tcp-ssh = final.callPackage ../../../user-apps/tcp-ssh {}; +(final: _prev: { + tcp-ssh = final.callPackage ../../../packages/tcp-ssh {}; }) diff --git a/user-apps/tcp-ssh/default.nix b/packages/tcp-ssh/default.nix similarity index 91% rename from user-apps/tcp-ssh/default.nix rename to packages/tcp-ssh/default.nix index e8388a6984..20ac8668d7 100644 --- a/user-apps/tcp-ssh/default.nix +++ b/packages/tcp-ssh/default.nix @@ -4,14 +4,12 @@ stdenvNoCC, pkgs, lib, - stdenv, ... }: let tcp-ssh = pkgs.writeShellScript "tcp-ssh" '' - #!/bin/bash # Check if two arguments (HOST and PORT) are provided if [ $# -ne 2 ]; then @@ -21,9 +19,9 @@ HOST="$1" PORT="$2" - PID_FILE="/tmp/tcp-ssh-$PORT.pid" + PID_FILE="''${PID_PATH:-./}tcp-ssh-$PORT.pid" - # Stop any running tcp-ssh tunnel instances + # Stop any running tcp-ssh tunnel instances for the same tcp port if [ -f "$PID_FILE" ]; then # TODO Killing PID(s) in a file, potential for misuse ${pkgs.procps}/bin/pkill -F "$PID_FILE" @@ -38,7 +36,7 @@ -o StrictHostKeyChecking=no \ -o ExitOnForwardFailure=yes \ -L $PORT:127.0.0.1:$PORT $HOST & - # Chatch the last command process (ssh) PID + # Catch the last command process (ssh) PID PID="$!" # Sleep for a second to verify connection does not die immediately diff --git a/targets/lenovo-x1-carbon.nix b/targets/lenovo-x1-carbon.nix index 68b44766e1..6266611225 100644 --- a/targets/lenovo-x1-carbon.nix +++ b/targets/lenovo-x1-carbon.nix @@ -31,28 +31,36 @@ } ]; audio.pciDevices = [ + # Passthrough of Intel Audio device 8086:51ca + # With the current implementation, the whole PCI IOMMU group 14: + # 00:1f.x in the example from Lenovo X1 Carbon + # must be defined for passthrough to AudioVM + + # ISA bridge: Intel Corporation Raptor Lake LPC/eSPI Controller (rev 01) { path = "0000:00:1f.0"; vendorId = "8086"; productId = "5194"; } + # Audio device: Intel Corporation Raptor Lake-P/U/H cAVS (rev 01) { path = "0000:00:1f.3"; vendorId = "8086"; productId = "51ca"; } + # SMBus: Intel Corporation Alder Lake PCH-P SMBus Host Controller (rev 01) { path = "0000:00:1f.4"; vendorId = "8086"; productId = "51a3"; } + # Serial bus controller: Intel Corporation Alder Lake-P PCH SPI Controller (rev 01) { path = "0000:00:1f.5"; vendorId = "8086"; productId = "51a4"; } ]; - }; virtioInputHostEvdevs = [ # Lenovo X1 touchpad and keyboard "/dev/input/by-path/platform-i8042-serio-0-event-kbd" @@ -182,11 +190,65 @@ ]; time.timeZone = "Asia/Dubai"; + # Tcp-ssh service runs in the GUIVM and makes ssh tunnel from localhost:4713 to AudioVM for pulseaudio + systemd.services.pulse-ssh = { + enable = true; + description = "tcp-ssh -tunnnel for pulseaudio"; + after = ["network-online.target"]; + serviceConfig = { + Type = "forking"; + Environment = [ + "PULSE_SERVER=\"tcp:localhost:4713\"" + "PID_PATH=\"/run/tcp-ssh/\"" + ]; + PIDFile = "/run/tcp-ssh/tcp-ssh-4713.pid"; + ExecStart = "${pkgs.tcp-ssh}/bin/tcp-ssh audio-vm.ghaf 4713"; + ExecStop = "${pkgs.procps}/bin/pkill -F /run/tcp-ssh/tcp-ssh-4713.pid"; + Restart = "on-failure"; + RestartSec = 5; + User = "ghaf"; + }; + wantedBy = ["ghaf-session.target"]; + }; + systemd.tmpfiles.rules = ["d /run/tcp-ssh 0750 ghaf ghaf -"]; }) ]; - audiovmConfig = hostConfiguration.config.ghaf.virtualization.microvm.audiovm; - audiovmExtraModules = []; + # Currently not needed + # audiovmConfig = hostConfiguration.config.ghaf.virtualization.microvm.audiovm; + audiovmExtraModules = [ + ({pkgs, ...}: { + microvm = { + shares = [ + { + tag = "waypipe-ssh-public-key"; + source = "/run/waypipe-ssh-public-key"; + mountPoint = "/run/waypipe-ssh-public-key"; + } + ]; + }; + fileSystems."/run/waypipe-ssh-public-key".options = ["ro"]; + # Waypipe-ssh key is used here to create keys for ssh tunneling to + # forward pulseaudio tcp connections. + # SSH is very picky about to file permissions and ownership and will + # accept neither direct path inside /nix/store or symlink that points + # there. Therefore we copy the file to /etc/ssh/get-auth-keys (by + # setting mode), instead of symlinking it. + environment.etc."ssh/get-auth-keys" = { + source = let + script = pkgs.writeShellScriptBin "get-auth-keys" '' + [[ "$1" != "ghaf" ]] && exit 0 + ${pkgs.coreutils}/bin/cat /run/waypipe-ssh-public-key/id_ed25519.pub + ''; + in "${script}/bin/get-auth-keys"; + mode = "0555"; + }; + services.openssh = { + authorizedKeysCommand = "/etc/ssh/get-auth-keys"; + authorizedKeysCommandUser = "nobody"; + }; + }) + ]; hostConfiguration = lib.nixosSystem { inherit system; specialArgs = {inherit lib;}; @@ -225,6 +287,26 @@ systemd.services."microvm@chromium-vm".after = ["microvm@audio-vm.service"]; systemd.services."microvm@chromium-vm".requires = ["microvm@audio-vm.service"]; + # Enable pulseaudio support for host as a service + sound.enable = true; + hardware.pulseaudio.enable = true; + hardware.pulseaudio.systemWide = true; + + # Allow microvm user to access pulseaudio + users.extraUsers.microvm.extraGroups = ["audio" "pulse-access"]; + + hardware.pulseaudio.extraConfig = '' + load-module module-null-sink sink_name=rtp sink_properties="device.description='RTP'" + # RTP Output to audio-vm for audio playback + load-module module-rtp-send source=rtp.monitor destination_ip=audio-vm.lan port=9875 + ''; + + hardware.pulseaudio.daemon.config = { + default-sample-rate = 44100; + alternate-sample-rate = 48000; + avoid-resampling = "yes"; + }; + ghaf = { hardware.definition = hwDefinition; host.kernel_hardening.enable = false; @@ -299,7 +381,6 @@ ] ++ audiovmExtraModules; }; - }; virtualization.microvm.appvm = { enable = true; vms = [ @@ -316,14 +397,17 @@ hardware.pulseaudio.enable = true; users.extraUsers.ghaf.extraGroups = ["audio"]; -<<<<<<< HEAD time.timeZone = "Asia/Dubai"; -======= hardware.pulseaudio.extraConfig = '' load-module module-combine-sink set-sink-volume @DEFAULT_SINK@ 60000 ''; ->>>>>>> bd4d3fd (Initial setup for AudioVM with passthrough for X1) + + hardware.pulseaudio.daemon.config = { + default-sample-rate = 44100; + alternate-sample-rate = 48000; + avoid-resampling = "yes"; + }; microvm.qemu.extraArgs = [ # Lenovo X1 integrated usb webcam @@ -333,7 +417,7 @@ "usb-host,vendorid=0x04f2,productid=0xb751" # Connect sound device to hosts pulseaudio socket "-audiodev" - "pa,id=pa1,server=tcp:192.168.101.4" + "pa,id=pa1,server=unix:/run/pulse/native" # Add HDA sound device to guest "-device" "intel-hda"