Google and Featherless support (#1041)

common/adapters.ts

@@ -248,6 +248,22 @@ export const OPENAI_CHAT_MODELS: Record<string, boolean> = {
   [OPENAI_MODELS.O1_Mini_20240912]: true,
 }
 
+export type GoogleModel = keyof typeof GOOGLE_MODELS
+
+export const GOOGLE_MODELS = {
+  GEMINI_15_PRO: { id: 'gemini-1.5-pro', label: 'Gemini 1.5 Pro' },
+  GEMINI_10_PRO_LATEST: { id: 'gemini-1.0-pro-latest', label: 'Gemini 1.0 Pro' },
+  GEMINI_15_FLASH: { id: 'gemini-1.5-flash', label: 'Gemini 1.5 Flash' },
+  GEMINI_15_FLASH_8B: { id: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash 8B' },
+}
+
+export const GOOGLE_LIMITS: Record<string, number> = {
+  'gemini-1.5-pro': 2097152,
+  'gemini-1.0-pro-latest': 32768,
+  'gemini-1.5-flash': 1048576,
+  'gemini-1.5-flash-8b': 1048576,
+}
+
 /** Note: claude-v1 and claude-instant-v1 not included as they may point
  * to different models in the future. New models may be less appropriate
  * for roleplaying so they should be updated to manually

common/presets.ts

@@ -56,6 +56,8 @@ export const presetValidator = {
   gaslight: 'string?',
   oaiModel: 'string',
   openRouterModel: 'any?',
+  featherlessModel: 'string?',
+  googleModel: 'string?',
 
   mirostatTau: 'number?',
   mirostatLR: 'number?',

common/presets/templates.ts

@@ -24,7 +24,7 @@ export type FormatTags = {
   closeSystem: string
 }
 
-export type ModelFormat = 'Alpaca' | 'Vicuna' | 'ChatML' | 'Mistral' | 'Llama3'
+export type ModelFormat = 'Alpaca' | 'Vicuna' | 'ChatML' | 'Mistral' | 'Llama3' | 'None'
 
 export const BUILTIN_FORMATS: { [key in ModelFormat]: FormatTags } = {
   Alpaca: {
@@ -67,6 +67,14 @@ export const BUILTIN_FORMATS: { [key in ModelFormat]: FormatTags } = {
     openBot: `<|start_header_id|>assistant<|end_header_id|>`,
     closeBot: `<|eot_id|>`,
   },
+  None: {
+    openSystem: '',
+    closeSystem: '',
+    openUser: '',
+    closeUser: '',
+    openBot: '',
+    closeBot: '',
+  },
 }
 
 export function replaceTags(prompt: string, format: FormatTags | ModelFormat) {

common/prompt-order.ts

@@ -1,6 +1,13 @@
 import { AppSchema } from './types'
 import { neat } from './util'
 
+export type OrderOptions = {
+  format?: string
+  order?: NonNullable<AppSchema.GenSettings['promptOrder']>
+  gen?: AppSchema.UserGenPreset
+  char?: AppSchema.Character
+}
+
 export function promptOrderToTemplate(
   format: string,
   order: NonNullable<AppSchema.GenSettings['promptOrder']>
@@ -27,10 +34,57 @@ export function promptOrderToTemplate(
     .trim()
 }
 
+export function promptOrderToSections(opts: OrderOptions) {
+  const order = (opts.order || SIMPLE_ORDER).filter(
+    (o) =>
+      o.placeholder !== 'system_prompt' &&
+      o.placeholder !== 'ujb' &&
+      o.placeholder !== 'post' &&
+      o.placeholder !== 'history' &&
+      o.enabled
+  )
+  const holders = opts.format
+    ? formatHolders[opts.format] || formatHolders.Universal
+    : formatHolders.Universal
+
+  const system = holders.system
+  const defs = order.map((o) => holders[o.placeholder]).join('\n')
+  const history = holders.history
+  const post = holders.post
+
+  return {
+    system,
+    defs: `<user>${defs}</user>`,
+    history,
+    post,
+  }
+}
+
+// export function promptOrderToMessages(opts: OrderOptions) {
+//   const sections = promptOrderToSections(opts)
+
+//   return [
+//     { role: 'system', content: sections.system },
+//     { role: 'user', content: sections.defs },
+
+//   ]
+// }
+
 function getOrderHolder(format: string, holder: string) {
   return formatHolders[format]?.[holder] || formatHolders.Universal[holder] || ''
 }
 
+export const SIMPLE_ORDER: NonNullable<AppSchema.GenSettings['promptOrder']> = [
+  'system_prompt',
+  'scenario',
+  'personality',
+  'impersonating',
+  'chat_embed',
+  'memory',
+  'example_dialogue',
+  'history',
+].map((placeholder) => ({ placeholder, enabled: true }))
+
 export const formatHolders: Record<string, Record<string, string>> = {
   Universal: {
     system_prompt: neat`<system>{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}</system>`,

common/prompt.ts

@@ -1,14 +1,20 @@
 import type { GenerateRequestV2 } from '../srv/adapter/type'
 import type { AppSchema, TokenCounter } from './types'
-import { AIAdapter, NOVEL_MODELS, OPENAI_CONTEXTS, THIRDPARTY_HANDLERS } from './adapters'
+import {
+  AIAdapter,
+  GOOGLE_LIMITS,
+  NOVEL_MODELS,
+  OPENAI_CONTEXTS,
+  THIRDPARTY_HANDLERS,
+} from './adapters'
 import { formatCharacter } from './characters'
 import { defaultTemplate } from './mode-templates'
 import { buildMemoryPrompt } from './memory'
 import { defaultPresets, getFallbackPreset, isDefaultPreset } from './presets'
 import { parseTemplate } from './template-parser'
 import { getMessageAuthor, getBotName, trimSentence, neat } from './util'
 import { Memory } from './types'
-import { promptOrderToTemplate } from './prompt-order'
+import { promptOrderToTemplate, SIMPLE_ORDER } from './prompt-order'
 import { ModelFormat, replaceTags } from './presets/templates'
 
 export type TickHandler<T = any> = (response: string, state: InferenceState, json?: T) => void
@@ -292,7 +298,7 @@ export function getTemplate(opts: Pick<GenerateRequestV2, 'settings' | 'chat'>)
   const fallback = getFallbackPreset(opts.settings?.service!)
   if (opts.settings?.useAdvancedPrompt === 'basic' || opts.settings?.presetMode === 'simple') {
     if (opts.settings.presetMode === 'simple') {
-      const template = promptOrderToTemplate('Universal', simpleOrder)
+      const template = promptOrderToTemplate('Universal', SIMPLE_ORDER)
       return template
     }
 
@@ -322,17 +328,6 @@ type InjectOpts = {
   encoder: TokenCounter
 }
 
-const simpleOrder: NonNullable<AppSchema.GenSettings['promptOrder']> = [
-  'system_prompt',
-  'scenario',
-  'personality',
-  'impersonating',
-  'chat_embed',
-  'memory',
-  'example_dialogue',
-  'history',
-].map((placeholder) => ({ placeholder, enabled: true }))
-
 export async function injectPlaceholders(template: string, inject: InjectOpts) {
   const { opts, parts, history: hist, encoder, ...rest } = inject
 
@@ -898,10 +893,22 @@ export function getContextLimit(
     // Any LLM could be used here so don't max any assumptions
     case 'ooba':
     case 'petals':
-    case 'kobold':
     case 'horde':
       return configuredMax - genAmount
 
+    case 'kobold': {
+      if (!gen.useMaxContext) return configuredMax - genAmount
+      switch (gen.thirdPartyFormat) {
+        case 'gemini': {
+          const max = GOOGLE_LIMITS[gen.googleModel!]
+          return max ? max - genAmount : configuredMax - genAmount
+        }
+
+        default:
+          return configuredMax - genAmount
+      }
+    }
+
     case 'novel': {
       const model = gen?.novelModel || NOVEL_MODELS.kayra_v1
       if (model === NOVEL_MODELS.clio_v1 || model === NOVEL_MODELS.kayra_v1) {

common/types/presets.ts

@@ -150,6 +150,8 @@ export interface GenSettings {
   claudeModel?: string
   mistralModel?: string
   openRouterModel?: OpenRouterModel
+  googleModel?: string
+  featherlessModel?: string
 
   thirdPartyUrl?: string
   thirdPartyFormat?: ThirdPartyFormat

package.json

@@ -104,6 +104,7 @@
     "multer": "1.4.5-lts.1",
     "nai-js-tokenizer": "1.0.1",
     "needle": "^3.2.0",
+    "partial-json": "^0.1.7",
     "patreon-api.ts": "^0.1.0",
     "peggy": "^3.0.2",
     "pino": "^8.10.0",

srv/adapter/agnaistic.ts

@@ -27,6 +27,7 @@ import { handleVenus } from './venus'
 import { sanitise, sanitiseAndTrim, trimResponseV2 } from '/common/requests/util'
 import { obtainLock, releaseLock } from '../api/chat/lock'
 import { getServerConfiguration } from '../db/admin'
+import { handleGemini } from './gemini'
 
 export type SubscriptionPreset = Awaited<NonNullable<ReturnType<typeof getSubscriptionPreset>>>
 
@@ -416,7 +417,13 @@ export function getHandlers(settings: Partial<AppSchema.GenSettings>) {
     case 'kobold':
     case 'openai':
       return handlers[settings.thirdPartyFormat!]
+
+    case 'featherless':
+      return handlers.kobold
+
+    case 'gemini':
+      return handleGemini
   }
 
-  return handlers.ooba
+  return handleThirdParty
 }

srv/adapter/featherless.ts

@@ -0,0 +1,51 @@
+import { logger } from '../middleware'
+
+export type FeatherlessModel = {
+  id: string
+  created_at: string
+  updated_at: string
+  name: string
+  owned_by: string
+  model_class: string
+  favorites: number
+  downloads: number
+  status: 'active' | 'not_deployed' | 'pending_deploy'
+  health?: 'OFFLINE' | 'UNHEALTHY' | 'HEALTHY'
+  avg_rating: number
+  total_reviews: number
+}
+
+let modelCache: FeatherlessModel[] = []
+
+export function getFeatherModels() {
+  return modelCache
+}
+
+async function getModelList() {
+  const res = await fetch('https://api.featherless.ai/feather/models?page=1&perPage=5000', {
+    headers: {
+      accept: '*/*',
+    },
+    method: 'GET',
+  })
+
+  if (res.status && res.status > 200) {
+    return
+  }
+
+  try {
+    const json = (await res.json()) as { items: FeatherlessModel[] }
+
+    if (json.items.length) {
+      modelCache = json.items
+    }
+
+    return json
+  } catch (ex) {
+    logger.warn({ err: ex, body: res.body, status: res.status }, `Featherless model list failed`)
+  }
+}
+
+getModelList()
+
+setInterval(getModelList, 120000)