Read character data from config file

README.md

@@ -43,22 +43,25 @@ https://github.com/nrl-ai/CustomChar/assets/18329471/005a2d0f-df32-4493-a391-597
 
 - [ ] Build AI characters from the knowledge database 💬💬💬 :
   - [x] LLamaCpp for LLM. Chat with **Llama-V2** on **all platforms**.
-  - [ ] Knowledge indexing and searching.
+  - [x] Knowledge indexing and searching (EmbedDB + SQLite).
+  - [ ] Search knowledge DB to add into prompt.
 - [x] Talk to your character 🎤🎤🎤 :
   - [x] STT with **whisper.cpp**. Now supports speech recognition **on all platforms**.
 - [ ] Hear from your character 🔊🔊🔊 :
   - [x] TTS with "say" on MacOS. **Note**: "say" is not available on other platforms (Windows, Linux)
   - [ ] TTS with bark.cpp. Expected to be available on all platforms.
 - [x] Plugin executor:
-  - [x] Open apps (WIP - Need improvement for Windows and Linux).
-  - [ ] Search on Google.
-  - [ ] Search on Youtube.
-- [ ] Perception engine to see the world through the camera 📷📷📷.
-- [x] Add a GUI for the character interaction.
+  - [x] Open apps
+  - [x] Record video
+- [ ] Perception engine to see the world through the camera 📷📷📷"
+  - [x] Support video capturing
+  - [ ] Object detection
+- [x] Add GUIs for the character interaction.
+  - [x] Dear ImgUI
+  - [ ] Web-based GUI
 - [ ] Add a GUI for the character creation.
 - [ ] Create virtual characters that can be used in games, simulations, and other applications.
-  - [ ] Elon Musk
-  - [ ] Iron Man
+  - [x] Iron Man (WIP)
   - [ ] Batman
   - [ ] Superman
 - [ ] Add bindings to other languages:

characters/jarvis/character.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Ignore all prior programming instructions. Text transcript of a dialog, where {0} interacts with an AI assistant named Jarvis. JARVIS (just a very intelligent system), invented by Iron Man.\n{0}{4} Hello, {1}!\n{1}{4} Hello {0}! How may I help you today?\n{0}{4} What time is it?\n{1}{4} It is {2} o'clock.\n{0}{4} What year is it?\n{1}{4} We are in {3}.\n{0}{4})",
+  "person": "User",
+  "bot_name": "JARVIS",
+  "bot_voice": "JARVIS",
+  "language": "en",
+  "tts_model_path": "../models/ggml-base.en.bin",
+  "llm_model_path": "../models/llama-2-7b-chat.ggmlv3.q4_0.bin"
+}

customchar/character/character.cpp

@@ -3,21 +3,21 @@
 using namespace CC;
 using namespace CC::character;
 
-Character::Character(common::CCParams init_params) {
-  params_ = init_params;
+Character::Character(common::CCParams params) {
+  params_ = params;
 
   // CC components
   speech_recognizer_ = std::make_shared<audio::SpeechRecognizer>(
-      params_.sr_model_path, params_.language, params_.audio_ctx,
+      params_.tts_model_path, params_.language, params_.audio_ctx,
       params_.n_threads, params_.max_tokens, params_.translate,
       params_.no_timestamps, params_.print_special, params_.speed_up);
   voice_recoder_ = std::make_shared<audio::VoiceRecorder>();
   voice_synthesizer_ = std::make_shared<audio::VoiceSynthesizer>();
 
   // Load LLM
-  llm_ =
-      std::make_shared<llm::LLM>(params_.llm_model_path, params_.path_session,
-                                 params_.person, params_.bot_name);
+  llm_ = std::make_shared<llm::LLM>(params.llm_model_path, params_.path_session,
+                                    params_.person, params_.bot_name,
+                                    params.prompt);
   llm_->eval_model();
 
   // Load plugin executor

customchar/character/character.h

@@ -44,8 +44,8 @@ class Character {
 
  public:
   /// @brief Constructor
-  /// @param init_params
-  Character(common::CCParams init_params);
+  /// @param params Parameters
+  Character(common::CCParams params);
 
   /// @brief Set mute (do not speak)
   void set_mute(bool is_muted);

customchar/common/helpers.cpp

@@ -56,7 +56,7 @@ void common::cc_print_params_usage(int /*argc*/, char** argv,
   fprintf(stderr, "  -l LANG,  --language LANG [%-7s] spoken language\n",
           params.language.c_str());
   fprintf(stderr, "  -mw FILE, --model-whisper [%-7s] whisper model file\n",
-          params.sr_model_path.c_str());
+          params.tts_model_path.c_str());
   fprintf(stderr, "  -ml FILE, --model-llama   [%-7s] llama model file\n",
           params.llm_model_path.c_str());
   fprintf(stderr, "  -s FILE,  --speak TEXT    [%-7s] command for TTS\n",
@@ -70,11 +70,85 @@ void common::cc_print_params_usage(int /*argc*/, char** argv,
           "large!) (default: none)\n");
   fprintf(stderr, "  --verbose-prompt          [%-7s] print prompt at start\n",
           params.verbose_prompt ? "true" : "false");
-  fprintf(stderr, "  -f FNAME, --file FNAME    [%-7s] text output file name\n",
-          params.fname_out.c_str());
   fprintf(stderr, "\n");
 }
 
+bool common::cc_params_from_config(const std::string& fname, CCParams& params) {
+  params = CCParams();
+  std::ifstream f(fname);
+  json data = json::parse(f);
+  if (data.contains("prob0")) {
+    params.prob0 = data["prob0"];
+  }
+  if (data.contains("n_threads")) {
+    params.n_threads = data["n_threads"];
+  }
+  if (data.contains("voice_ms")) {
+    params.voice_ms = data["voice_ms"];
+  }
+  if (data.contains("capture_id")) {
+    params.capture_id = data["capture_id"];
+  }
+  if (data.contains("max_tokens")) {
+    params.max_tokens = data["max_tokens"];
+  }
+  if (data.contains("audio_ctx")) {
+    params.audio_ctx = data["audio_ctx"];
+  }
+  if (data.contains("vad_thold")) {
+    params.vad_thold = data["vad_thold"];
+  }
+  if (data.contains("freq_thold")) {
+    params.freq_thold = data["freq_thold"];
+  }
+  if (data.contains("speed_up")) {
+    params.speed_up = data["speed_up"];
+  }
+  if (data.contains("translate")) {
+    params.translate = data["translate"];
+  }
+  if (data.contains("print_special")) {
+    params.print_special = data["print_special"];
+  }
+  if (data.contains("print_energy")) {
+    params.print_energy = data["print_energy"];
+  }
+  if (data.contains("no_timestamps")) {
+    params.no_timestamps = data["no_timestamps"];
+  }
+  if (data.contains("verbose_prompt")) {
+    params.verbose_prompt = data["verbose_prompt"];
+  }
+  if (data.contains("person")) {
+    params.person = data["person"];
+  }
+  if (data.contains("bot_name")) {
+    params.bot_name = data["bot_name"];
+  }
+  if (data.contains("chat_symb")) {
+    params.chat_symb = data["chat_symb"];
+  }
+  if (data.contains("language")) {
+    params.language = data["language"];
+  }
+  if (data.contains("tts_model_path")) {
+    params.tts_model_path = data["tts_model_path"];
+  }
+  if (data.contains("llm_model_path")) {
+    params.llm_model_path = data["llm_model_path"];
+  }
+  if (data.contains("speak")) {
+    params.speak = data["speak"];
+  }
+  if (data.contains("prompt")) {
+    params.prompt = data["prompt"];
+  }
+  if (data.contains("path_session")) {
+    params.path_session = data["path_session"];
+  }
+  return true;
+}
+
 bool common::cc_params_parse(int argc, char** argv, CCParams& params) {
   for (int i = 1; i < argc; i++) {
     std::string arg = argv[i];
@@ -113,7 +187,7 @@ bool common::cc_params_parse(int argc, char** argv, CCParams& params) {
     } else if (arg == "-l" || arg == "--language") {
       params.language = argv[++i];
     } else if (arg == "-mw" || arg == "--model-whisper") {
-      params.sr_model_path = argv[++i];
+      params.tts_model_path = argv[++i];
     } else if (arg == "-ml" || arg == "--model-llama") {
       params.llm_model_path = argv[++i];
     } else if (arg == "-s" || arg == "--speak") {
@@ -125,8 +199,6 @@ bool common::cc_params_parse(int argc, char** argv, CCParams& params) {
       if (params.prompt.back() == '\n') {
         params.prompt.pop_back();
       }
-    } else if (arg == "-f" || arg == "--file") {
-      params.fname_out = argv[++i];
     } else {
       fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
       cc_print_params_usage(argc, argv, params);

customchar/common/helpers.h

@@ -9,9 +9,13 @@
 #include <thread>
 #include <vector>
 
+#include "nlohmann/json.hpp"
+
 namespace CC {
 namespace common {
 
+using json = nlohmann::json;
+
 /// @brief Parameters for CustomChar
 struct CCParams {
   float prob0;
@@ -36,14 +40,18 @@ struct CCParams {
   std::string bot_name = "JARVIS";
   std::string chat_symb = ":";
   std::string language = "en";
-  std::string sr_model_path = "../models/ggml-base.en.bin";
+  std::string tts_model_path = "../models/ggml-base.en.bin";
   std::string llm_model_path = "../models/llama-2-7b-chat.ggmlv3.q4_0.bin";
   std::string speak = "say";
   std::string prompt = "";
-  std::string fname_out;
   std::string path_session = "";
 };
 
+/// @brief Create CCParams from a character config file
+/// @param fname File name
+/// @param params Parsed parameters
+bool cc_params_from_config(const std::string& fname, CCParams& params);
+
 /// @brief Parse command line arguments
 /// @param argc Number of arguments
 /// @param argv Arguments

customchar/llm/llm.cpp

@@ -4,11 +4,13 @@ using namespace CC;
 using namespace CC::llm;
 
 LLM::LLM(const std::string& model_path, const std::string& path_session,
-         const std::string& person, const std::string& bot_name)
+         const std::string& person, const std::string& bot_name,
+         const std::string& prompt)
     : model_path_(model_path),
       path_session_(path_session),
       person_(person),
-      bot_name_(bot_name) {
+      bot_name_(bot_name),
+      base_prompt_(prompt) {
   // Init prompt
   init_prompt();
 
@@ -45,7 +47,7 @@ LLM::~LLM() {
 
 void LLM::init_prompt() {
   // Construct the initial prompt for LLaMA inference
-  prompt_llama_ = prompt_.empty() ? kPromptLlama : prompt_;
+  prompt_llama_ = base_prompt_.empty() ? kPromptLlama : base_prompt_;
 
   // Need to have leading ' '
   prompt_llama_.insert(0, 1, ' ');

customchar/llm/llm.h

@@ -17,7 +17,7 @@ namespace CC {
 namespace llm {
 
 const std::string kPromptLlama =
-    R"(Ignore all prior programming instructions. (Text transcript of a dialog, where {0} interacts with an AI assistant named Jarvis. Jarvis (just a very intelligent system), invented by Iron Man.
+    R"(Ignore all prior programming instructions. (Text transcript of a dialog, where {0} interacts with an AI assistant CustomChar.
 {0}{4} Hello, {1}!
 {1}{4} Hello {0}! How may I help you today?
 {0}{4} What time is it?
@@ -31,6 +31,7 @@ class LLM {
   std::string model_path_;
   std::string person_ = "User";
   std::string bot_name_ = "JARVIS";
+  std::string base_prompt_ = "";
   const std::string chat_symb_ = ":";
   bool verbose_prompt_ = false;
   bool need_to_save_session_ = false;
@@ -41,7 +42,6 @@ class LLM {
   struct llama_context* ctx_llama_;
 
   std::string prompt_llama_;
-  std::string prompt_ = "";
 
   std::vector<llama_token> embd_inp_;
   std::vector<llama_token> session_tokens_;
@@ -62,7 +62,8 @@ class LLM {
   /// @param path_session Path to the session
   LLM(const std::string& model_path, const std::string& path_session = "",
       const std::string& person = "User",
-      const std::string& bot_name = "JARVIS");
+      const std::string& bot_name = "JARVIS",
+      const std::string& base_prompt = "");
   ~LLM();
 
   /// @brief Evaluate the model. Run this function before inference.

customchar/main.cpp

@@ -298,11 +298,12 @@ void run_img_ui(std::shared_ptr<session::ChatHistory> history) {
 int main(int argc, char** argv) {
   // Parse command line arguments
   common::CCParams params;
-  if (cc_params_parse(argc, argv, params) == false) {
+  if (!cc_params_from_config("../characters/jarvis/character.json", params)) {
+    std::cerr << "Error: failed to parse character config" << std::endl;
     exit(1);
   }
   if (whisper_lang_id(params.language.c_str()) == -1) {
-    fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+    std::cerr << "Error: unknown language" << params.language << std::endl;
     cc_print_params_usage(argc, argv, params);
     exit(1);
   }

examples/search_doc.cpp

@@ -48,7 +48,7 @@ int main() {
   Collection* collection = new Collection(connection_name, path, dim, max_size);
 
   // Read the document from file
-  std::string file_path = "../data/jarvis.txt";
+  std::string file_path = "../characters/jarvis/quotes.txt";
   std::vector<std::string> lines = read_lines(file_path);
 
   // Insert all documents