Add Falcon LLM

Signed-off-by: Humaid Alqasimi <[email protected]>

modules/common/services/desktop.nix

@@ -180,6 +180,13 @@ in
               icon = "${pkgs.icon-pack}/preferences-sound.svg";
             }
 
+            {
+              name = "Falcon AI";
+              description = "Your local large language model, developed by TII.";
+              path = "${pkgs.alpaca}/bin/alpaca";
+              icon = "${pkgs.ghaf-artwork}/icons/falcon-icon.svg";
+            }
+
             {
               name = "Shutdown";
               description = "Shutdown System";

modules/microvm/virtualization/microvm/guivm.nix

@@ -75,6 +75,12 @@ let
                   group = config.ghaf.users.accounts.user;
                   mode = "u=rwx,g=,o=";
                 }
+                {
+                  directory = "/var/lib/private/ollama";
+                  inherit (config.ghaf.users.accounts) user;
+                  group = "ollama";
+                  mode = "u=rwx,g=,o=";
+                }
               ];
               users.${config.ghaf.users.accounts.user}.directories = [
                 ".config/"
@@ -153,7 +159,7 @@ let
           microvm = {
             optimize.enable = false;
             vcpu = 2;
-            mem = 2048;
+            mem = 12288;
             hypervisor = "qemu";
             shares = [
               {
@@ -188,8 +194,11 @@ let
           imports = [
             ../../../common
             ../../../desktop
+            ../../../reference/services
           ];
 
+          ghaf.reference.services.ollama = true;
+
           # Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs
           systemd.user.services.waypipe = {
             enable = true;

modules/reference/services/default.nix

@@ -10,10 +10,12 @@ in
   imports = [
     ./dendrite-pinecone/dendrite-pinecone.nix
     ./dendrite-pinecone/dendrite-config.nix
+    ./ollama/ollama.nix
   ];
   options.ghaf.reference.services = {
-    enable = mkEnableOption "Enable the Ghaf reference services";
-    dendrite = mkEnableOption "Enable the dendrite-pinecone service";
+    enable = mkEnableOption "Ghaf reference services";
+    dendrite = mkEnableOption "dendrite-pinecone service";
+    ollama = mkEnableOption "ollama service";
   };
   config = mkIf cfg.enable {
     ghaf.reference.services = {

modules/reference/services/ollama/ollama.nix

@@ -0,0 +1,77 @@
+# Copyright 2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{ config, lib, ... }:
+let
+  cfg = config.ghaf.reference.services;
+  inherit (lib) mkIf;
+in
+{
+  config = mkIf cfg.ollama {
+    services.ollama = {
+      enable = true;
+      openFirewall = true;
+      host = "0.0.0.0";
+      loadModels = [ "falcon2" ];
+    };
+
+    # This forces Alpaca to use the systemd ollama daemon instead of spawning
+    # its own.
+    system.userActivationScripts.alpaca-configure = {
+      text = ''
+                source ${config.system.build.setEnvironment}
+                mkdir -p $HOME/.config/com.jeffser.Alpaca
+                cat <<EOF > $HOME/.config/com.jeffser.Alpaca/server.json
+        {
+              "remote_url": "http://localhost:11434",
+              "remote_bearer_token": "",
+              "run_remote": true,
+              "local_port": 11435,
+              "run_on_background": false,
+              "powersaver_warning": true,
+              "model_tweaks": {
+                    "temperature": 0.7,
+                    "seed": 0,
+                    "keep_alive": 5
+              },
+              "ollama_overrides": {},
+              "idle_timer": 0
+        }
+        EOF
+      '';
+    };
+
+    # Carried over from nixpkgs
+    # TODO remove when nixpkgs is bumped
+    systemd.services.ollama-model-loader = {
+      wantedBy = [
+        "multi-user.target"
+        "ollama.service"
+      ];
+      after = [ "ollama.service" ];
+      bindsTo = [ "ollama.service" ];
+      inherit (config.systemd.services.ollama) environment;
+      serviceConfig = {
+        Type = "exec";
+        DynamicUser = true;
+        Restart = "on-failure";
+        RestartSec = "1s";
+        RestartMaxDelaySec = "2h";
+        RestartSteps = "10";
+      };
+
+      script = ''
+        '${lib.getExe config.services.ollama.package}' pull falcon2:latest
+      '';
+    };
+
+    # TODO also remove this entirely after nixpkgs bump
+    systemd.services.ollama = {
+      postStart = lib.mkForce "";
+      serviceConfig = {
+        TimeoutStartSec = "5h";
+        Restart = "always";
+        RestartSec = "5s";
+      };
+    };
+  };
+}

overlays/custom-packages/alpaca/default.nix

@@ -0,0 +1,18 @@
+# Copyright 2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Bump to newer version of Alpaca due to a bug.
+#
+{ prev }:
+prev.alpaca.overrideAttrs rec {
+  version = "2.0.6";
+  src = prev.fetchFromGitHub {
+    owner = "Jeffser";
+    repo = "Alpaca";
+    rev = "refs/tags/${version}";
+    hash = "sha256-4c6pisd3o7mycivHd1QZ2N7s8pYzrQXiZMbVvl5ciPA=";
+  };
+
+  patches = [ ];
+  postPatch = "";
+}

overlays/custom-packages/default.nix

@@ -21,4 +21,6 @@
   gtklock = import ./gtklock { inherit prev; };
   hardware-scan = final.callPackage ../../packages/hardware-scan { };
   pulseaudio-ghaf = import ./pulseaudio { inherit prev; };
+  ollama = import ./ollama { inherit prev; };
+  alpaca = import ./alpaca { inherit prev; };
 })

overlays/custom-packages/ollama/default.nix

@@ -0,0 +1,15 @@
+# Copyright 2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Use latest version of ollama. Should be removed when nixpkgs is bumped.
+#
+{ prev }:
+prev.ollama.overrideAttrs {
+  src = prev.fetchFromGitHub {
+    owner = "ollama";
+    repo = "ollama";
+    rev = "v0.3.1";
+    hash = "sha256-ctz9xh1wisG0YUxglygKHIvU9bMgMLkGqDoknb8qSAU=";
+    fetchSubmodules = true;
+  };
+}