Embeddings (#509)

.github/workflows/dev-deploy.yml

@@ -40,11 +40,12 @@ jobs:
         env:
           INJECT_SCRIPT: ${{ secrets.INJECT_SCRIPT }}
         run: |
+          pnpm run build:server
           pnpm run build:prod
           cp dist/index.html dist/original.html
           node .github/inject.js
 
-      - name: Publish to Dev
+      - name: Publish to DEV
         uses: BetaHuhn/do-spaces-action@v2
         with:
           access_key: ${{ secrets.S3_ACCESS_KEY}}
@@ -53,3 +54,12 @@ jobs:
           space_region: ${{ secrets.S3_REGION }}
           permission: public-read
           source: dist
+
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          node .github/rev-version.js $GITHUB_SHA
+          npm publish

README.md

@@ -23,13 +23,13 @@ Agnaistic is bundled as an NPM package and can be installed globally:
 # View launch options:
 > agnai help
 
-# Run with the pipeline features (Coming soon!)
+# Run with the Pipeline features
 > agnai --pipeline
 
 ```
 
 When using the NPM package, your images and JSON files will be stored in: `HOME_FOLDER/.agnai`.  
-E.g. `/home/sceuick/.agnai/json` `/home/sceuick/.agnai/assets`
+E.g. Linux: `/home/sceuick/.agnai/` Mac: `/Users/sceuick/.agnai` Windows: `C:\Users\sceuick\.agnai`.
 
 ## Features
 
@@ -41,9 +41,9 @@ E.g. `/home/sceuick/.agnai/json` `/home/sceuick/.agnai/assets`
   - User settings: Which AI service to use and their own settings
   - User generation settings
 - Chat specific overrides: AI Service, Character, Generation Settings
-- **Optional pipeline features (Coming soon!)**
-  - Text summarization for images
-  - TODO: Long-term memory, image captioning, text-to-speech, ...
+- **Optional pipeline features**
+  - Long-term memory
+  - Wikipedia Article and PDF embedding
 
 ## Running with Docker
 
@@ -73,18 +73,6 @@ E.g. `/home/sceuick/.agnai/json` `/home/sceuick/.agnai/assets`
      - Mac/Linux: `npm run start:public`
      - Windows: `npm run start:public:win`
 
-## Design Goals
-
-This project quickly deviated from the upstream project. This project is not intended to be a SaaS nor be centered around the Pygmalion model.  
-Ultimately the design goals for this project are my own.
-
-- High quality codebase
-- AI Services: Transparently use a variety of AI models and services to converse with
-  - Initial AI services: Kobold, AI Horde, and Novel
-- Supporting additional AI services should be low friction
-- Lightweight to self-host
-- Avoiding native dependencies and Docker to be easy for non-technical people to install and run
-
 ## Self-Hosting Settings
 
 To try and cater for the small tweaks and tuning that people need for their specific needs at an application level we have `settings.json`.  
@@ -145,10 +133,13 @@ The important parts of the stack are:
 # Windows
 > npm run start:win
 
-# Install and run pipeline API
-> npm run model:init  # Install poetry into a virtual environment
-> npm run model:deps  # Install/update poetry dependencies
-> npm run model:start # Start the pipeline API
+# Install and run Pipeline API
+# If required, this will update the dependencies before running the API
+> npm run model # Install poetry into a virtual environment
+
+# Run everything with a single commmand:
+> npm run start:all # Linux and OSX
+> npm run start:all:win # Windows
 ```
 
 ### Recommended Developer Tooling

common/grammar.ts

@@ -81,6 +81,8 @@ Memory "memory" = "memory"i { return "memory" }
 Message "message" = "msg"i / "message"i / "text"i { return "message" }
 ChatAge "chat-age" = "chat_age"i { return "chat_age" }
 IdleDuration "idle-duration" = "idle_duration"i { return "idle_duration" }
+ChatEmbed "chat-embed" = "chat_embed"i { return "chat_embed" }
+UserEmbed "user-embed" = "user"i { return "user_embed" }
 
 // Iterable entities
 Bots "bots" = ( "bots"i / "characters"i ) { return "bots" }
@@ -100,4 +102,6 @@ Interp "interp"
     / Message
     / ChatAge
     / IdleDuration
+    / ChatEmbed
+    / UserEmbed
 `

common/presets.ts

@@ -16,8 +16,12 @@ export const chatGenSettings = {
   repetitionPenalty: 'number',
   repetitionPenaltyRange: 'number',
   repetitionPenaltySlope: 'number',
+
   memoryContextLimit: 'number?',
   memoryDepth: 'number?',
+  memoryChatEmbedLimit: 'number?',
+  memoryUserEmbedLimit: 'number?',
+
   typicalP: 'number',
   topP: 'number',
   topK: 'number',

common/presets/openai.ts

@@ -19,10 +19,14 @@ export const openaiPresets = {
     ignoreCharacterUjb: false,
     useGaslight: true,
     gaslight: `{{system_prompt}}
-Description of {{char}}:
-{{personality}}
+Description of {{char}}: {{personality}}
+
 Circumstances and context of the dialogue: {{scenario}}
-Facts: {{memory}}`,
+
+Facts: {{memory}}
+
+Relevant Information: {{user_embed}}
+`,
   },
   openaiAlt: {
     name: 'Turbo (#2)',
@@ -40,10 +44,14 @@ Facts: {{memory}}`,
     ignoreCharacterUjb: false,
     useGaslight: true,
     gaslight: `{{system_prompt}}
-Description of {{char}}:
-{{personality}}
+Description of {{char}}: {{personality}}
+
 Circumstances and context of the dialogue: {{scenario}}
-Facts: {{memory}}`,
+
+Facts: {{memory}}
+
+Relevant Information: {{user_embed}}
+`,
   },
   openaiTurbo: {
     name: 'DaVinci',
@@ -61,9 +69,10 @@ Facts: {{memory}}`,
     ignoreCharacterUjb: false,
     systemPrompt: `Enter roleplay mode. You will write {{char}}'s next reply in a dialogue between {{char}} and {{user}}. Do not decide what {{user}} says or does. Use Internet roleplay style, e.g. no quotation marks, and write user actions in italic in third person like: *example*. You are allowed to use markdown. Be proactive, creative, drive the plot and conversation forward. Write at least one paragraph, up to four. Always stay in character. Always keep the conversation going. (Repetition is highly discouraged)`,
     gaslight: `{{system_prompt}}
-Description of {{char}}:
-{{personality}}
+Description of {{char}}: {{personality}}
+
 Circumstances and context of the dialogue: {{scenario}}
+
 Facts: {{memory}}`,
   },
 } satisfies Record<string, Partial<AppSchema.GenSettings>>

common/prompt.ts

@@ -9,6 +9,7 @@ import { defaultPresets, getFallbackPreset, isDefaultPreset } from './presets'
 import { parseTemplate } from './template-parser'
 import { Encoder } from './tokenize'
 import { elapsedSince, trimSentence } from './util'
+import { Memory } from './types'
 
 export const SAMPLE_CHAT_MARKER = `System: New conversation started. Previous conversations are examples only.`
 export const SAMPLE_CHAT_PREAMBLE = `How {{char}} speaks:`
@@ -26,6 +27,9 @@ export type PromptParts = {
 
   /** User's impersonated personality */
   impersonality?: string
+
+  chatEmbeds: string[]
+  userEmbeds: string[]
 }
 
 export type Prompt = {
@@ -57,6 +61,8 @@ export type PromptOpts = {
   impersonate?: AppSchema.Character
   lastMessage: string
   trimSentences?: boolean
+  chatEmbeds: Memory.UserEmbed<{ name: string }>[]
+  userEmbeds: Memory.UserEmbed[]
 }
 
 type BuildPromptOpts = {
@@ -69,6 +75,8 @@ type BuildPromptOpts = {
   members: AppSchema.Profile[]
   settings?: Partial<AppSchema.GenSettings>
   impersonate?: AppSchema.Character
+  chatEmbed?: Memory.UserEmbed<{ name: string }>[]
+  userEmbed?: Memory.UserEmbed[]
 }
 
 /** {{user}}, <user>, {{char}}, <bot>, case insensitive */
@@ -90,6 +98,8 @@ const HOLDER_NAMES = {
   chatAge: 'chat_age',
   idleDuration: 'idle_duration',
   impersonating: 'impersonating',
+  chatEmbed: 'chat_embed',
+  userEmbed: 'user_embed',
 }
 
 export const HOLDERS = {
@@ -106,6 +116,8 @@ export const HOLDERS = {
   systemPrompt: /{{system_prompt}}/gi,
   linebreak: /{{(br|linebreak|newline)}}/gi,
   impersonating: /{{impersonating}}/gi,
+  chatEmbed: /{{chat_embed}}/gi,
+  userEmbed: /{{user_embed}}/gi,
 }
 
 const ALL_HOLDERS = new RegExp(
@@ -255,6 +267,8 @@ export function injectPlaceholders(template: string, { opts, parts, history: his
     .replace(HOLDERS.linebreak, '\n')
     .replace(HOLDERS.chatAge, elapsedSince(opts.chat.createdAt))
     .replace(HOLDERS.idleDuration, elapsedSince(rest.lastMessage || ''))
+    .replace(HOLDERS.chatEmbed, parts.chatEmbeds.join('\n') || '')
+    .replace(HOLDERS.userEmbed, parts.userEmbeds.join('\n') || '')
     // system prompt should not support other placeholders
     .replace(HOLDERS.systemPrompt, newline(parts.systemPrompt))
     // All placeholders support {{char}} and {{user}} placeholders therefore these must be last
@@ -279,6 +293,8 @@ function removeUnusedPlaceholders(template: string, parts: PromptParts) {
   const useScenario = !!parts.scenario
   const useSystemPrompt = !!parts.systemPrompt
   const useImpersonality = !!parts.impersonality
+  const useChatEmbed = parts.chatEmbeds.join('').length > 0
+  const useUserEmbed = parts.userEmbeds.join('').length > 0
 
   /**
    * Filter out lines that contain only one 'one of a kind' placeholder where the placeholder is empty
@@ -300,6 +316,8 @@ function removeUnusedPlaceholders(template: string, parts: PromptParts) {
       if (!useScenario && line.match(HOLDERS.scenario)) return false
       if (!useImpersonality && line.match(HOLDERS.impersonating)) return false
       if (!useSystemPrompt && line.match(HOLDERS.systemPrompt)) return false
+      if (!useChatEmbed && line.match(HOLDERS.chatEmbed)) return false
+      if (!useUserEmbed && line.match(HOLDERS.userEmbed)) return false
       return true
     })
     .join('\n')
@@ -310,17 +328,21 @@ function removeUnusedPlaceholders(template: string, parts: PromptParts) {
 export function ensureValidTemplate(
   template: string,
   parts: PromptParts,
-  skip?: Array<'history' | 'post' | 'persona' | 'scenario'>
+  skip?: Array<'history' | 'post' | 'persona' | 'scenario' | 'userEmbed' | 'chatEmbed'>
 ) {
   const skips = new Set(skip || [])
   let hasScenario = !!template.match(HOLDERS.scenario)
   let hasPersona = !!template.match(HOLDERS.persona)
   let hasHistory = !!template.match(HOLDERS.history)
   let hasPost = !!template.match(HOLDERS.post)
   let hasUjb = !!template.match(HOLDERS.ujb)
+  let hasUserEmbed = !!template.match(HOLDERS.userEmbed)
+  // let hasChatEmbed = !!template.match(HOLDERS.chatEmbed)
 
   const useScenario = !!parts.scenario
   const usePersona = !!parts.persona
+  const useUserEmbed = parts.userEmbeds.length > 0
+  // const useChatEmbed = parts.chatEmbeds.length > 0
 
   let modified = removeUnusedPlaceholders(template, parts)
 
@@ -334,6 +356,11 @@ export function ensureValidTemplate(
     modified += `\n{{char}}'s persona: {{${HOLDER_NAMES.persona}}}`
   }
 
+  if (!skips.has('userEmbed') && !hasUserEmbed && useUserEmbed) {
+    hasUserEmbed = true
+    modified += `\nRelevant Information: {{${HOLDER_NAMES.userEmbed}}}`
+  }
+
   if (!skips.has('post') && !skips.has('history') && !hasHistory && !hasPost) {
     modified += `\n{{history}}\n{{post}}`
   } else if (!skips.has('history') && !hasHistory && hasPost) {
@@ -361,6 +388,8 @@ type PromptPartsOptions = Pick<
   | 'replyAs'
   | 'impersonate'
   | 'characters'
+  | 'chatEmbeds'
+  | 'userEmbeds'
 >
 
 export function getPromptParts(opts: PromptPartsOptions, lines: string[], encoder: Encoder) {
@@ -378,6 +407,8 @@ export function getPromptParts(opts: PromptPartsOptions, lines: string[], encode
     ),
     post: [],
     allPersonas: [],
+    chatEmbeds: [],
+    userEmbeds: [],
   }
 
   const personalities = new Set([replyAs._id])
@@ -440,6 +471,18 @@ export function getPromptParts(opts: PromptPartsOptions, lines: string[], encode
 
   parts.post = post.map(replace)
 
+  if (opts.userEmbeds) {
+    const embeds = opts.userEmbeds.map((line) => line.text)
+    const fit = fillPromptWithLines(encoder, opts.settings?.memoryUserEmbedLimit || 500, '', embeds)
+    parts.userEmbeds = fit
+  }
+
+  if (opts.chatEmbeds) {
+    const embeds = opts.chatEmbeds.map((line) => `${line.name}: ${line.text}`)
+    const fit = fillPromptWithLines(encoder, opts.settings?.memoryChatEmbedLimit || 500, '', embeds)
+    parts.chatEmbeds = fit
+  }
+
   return parts
 }
 
@@ -607,7 +650,7 @@ export function getChatPreset(
   }
 
   // #4
-  const { adapter, isThirdParty } = getAdapter(chat, user)
+  const { adapter, isThirdParty } = getAdapter(chat, user, undefined)
   const fallbackId = user.defaultPresets?.[isThirdParty ? 'kobold' : adapter]
 
   if (fallbackId) {
@@ -627,7 +670,11 @@ export function getChatPreset(
  * 3. chat.adapter
  * 4. user.defaultAdapter
  */
-export function getAdapter(chat: AppSchema.Chat, config: AppSchema.User, preset?: Partial<AppSchema.GenSettings>) {
+export function getAdapter(
+  chat: AppSchema.Chat,
+  config: AppSchema.User,
+  preset: Partial<AppSchema.GenSettings> | undefined
+) {
   const chatAdapter = !chat.adapter || chat.adapter === 'default' ? config.defaultAdapter : chat.adapter
 
   let adapter = preset?.service ? preset.service : chatAdapter

common/template-parser.ts

@@ -1,7 +1,7 @@
 import { formatCharacter } from './characters'
 import { grammar } from './grammar'
 import { PromptParts } from './prompt'
-import { AppSchema } from '/common/types'
+import { AppSchema, Memory } from '/common/types'
 import peggy from 'peggy'
 import { elapsedSince } from './util'
 
@@ -37,6 +37,8 @@ type Holder =
   | 'chat_age'
   | 'idle_duration'
   | 'all_personalities'
+  | 'chat_embed'
+  | 'user_embed'
 
 type IterableHolder = 'history' | 'bots'
 
@@ -56,6 +58,8 @@ export type ParseOpts = {
   characters: Record<string, AppSchema.Character>
   sender: AppSchema.Profile
   lastMessage?: string
+  chatEmbed?: Memory.UserEmbed<{ name: string }>[]
+  userEmbed?: Memory.UserEmbed[]
 }
 
 export function parseTemplate(template: string, opts: ParseOpts) {
@@ -254,6 +258,12 @@ function getPlaceholder(value: Holder, opts: ParseOpts) {
 
     case 'all_personalities':
       return opts.parts.allPersonas?.join('\n') || ''
+
+    case 'chat_embed':
+      return opts.parts.chatEmbeds.join('\n') || ''
+
+    case 'user_embed':
+      return opts.parts.userEmbeds.join('\n') || ''
   }
 }
 

common/types/index.ts

@@ -1,6 +1,8 @@
 import * as UI from './ui'
 import * as Sprite from './sprite'
+import * as Memory from './memory'
 
 export * from './schema'
 export * from './texttospeech-schema'
-export { UI, Sprite }
+
+export { UI, Sprite, Memory }