Guidance max tokens (#518)

.prettierrc

@@ -1,5 +1,5 @@
 {
-  "printWidth": 120,
+  "printWidth": 100,
   "semi": false,
   "trailingComma": "es5",
   "singleQuote": true,

common/adapters.ts

@@ -237,7 +237,16 @@ export const adapterSettings: {
 
   gaslight: ['openai', 'novel', 'scale', 'kobold', 'claude', 'ooba', 'goose', 'openrouter'],
   systemPrompt: ['openai', 'novel', 'scale', 'kobold', 'claude', 'ooba', 'goose', 'openrouter'],
-  ignoreCharacterSystemPrompt: ['openai', 'novel', 'scale', 'kobold', 'claude', 'ooba', 'goose', 'openrouter'],
+  ignoreCharacterSystemPrompt: [
+    'openai',
+    'novel',
+    'scale',
+    'kobold',
+    'claude',
+    'ooba',
+    'goose',
+    'openrouter',
+  ],
   ultimeJailbreak: ['openai', 'claude', 'kobold', 'scale', 'openrouter'],
   ignoreCharacterUjb: ['openai', 'claude', 'kobold', 'openrouter'],
 

common/characters.ts

@@ -20,7 +20,10 @@ export const defaultChars = {
       "*A soft smile appears on {{char}}'s face as {{user}} enters the cafe and takes a seat* *Beep! Boop!* Hello, {{user}}! It's good to see you again. What would you like to chat about?",
   },
 } satisfies {
-  [key: string]: Pick<AppSchema.Character, 'name' | 'persona' | 'sampleChat' | 'scenario' | 'greeting'>
+  [key: string]: Pick<
+    AppSchema.Character,
+    'name' | 'persona' | 'sampleChat' | 'scenario' | 'greeting'
+  >
 }
 
 export function exportCharacter(char: AppSchema.Character, target: 'tavern' | 'ooba') {
@@ -53,7 +56,9 @@ export function exportCharacter(char: AppSchema.Character, target: 'tavern' | 'o
           system_prompt: char.systemPrompt ?? '',
           post_history_instructions: char.postHistoryInstructions ?? '',
           alternate_greetings: char.alternateGreetings ?? [],
-          character_book: char.characterBook ? nativeToCharacterBook(char.characterBook) : undefined,
+          character_book: char.characterBook
+            ? nativeToCharacterBook(char.characterBook)
+            : undefined,
           tags: char.tags ?? [],
           creator: char.creator ?? '',
           character_version: char.characterVersion ?? '',
@@ -80,7 +85,11 @@ export function exportCharacter(char: AppSchema.Character, target: 'tavern' | 'o
   }
 }
 
-export function formatCharacter(name: string, persona: AppSchema.Persona, kind?: AppSchema.Persona['kind']) {
+export function formatCharacter(
+  name: string,
+  persona: AppSchema.Persona,
+  kind?: AppSchema.Persona['kind']
+) {
   switch (kind || persona.kind) {
     case 'wpp': {
       const attrs = Object.entries(persona.attributes)
@@ -91,7 +100,9 @@ export function formatCharacter(name: string, persona: AppSchema.Persona, kind?:
     }
 
     case 'sbf': {
-      const attrs = Object.entries(persona.attributes).map(([key, values]) => `${key}: ${values.map(quote).join(', ')}`)
+      const attrs = Object.entries(persona.attributes).map(
+        ([key, values]) => `${key}: ${values.map(quote).join(', ')}`
+      )
 
       return `[ character: "${name}"; ${attrs.join('; ')} ]`
     }

common/default-preset.ts

@@ -75,7 +75,9 @@ export const altJailbreak = `
 
 export const cyoaTemplate = (model?: string) => {
   const jailbreak =
-    model === OPENAI_MODELS.Turbo0613 || model === OPENAI_MODELS.Turbo ? modernJailbreak : originalJailbreak
+    model === OPENAI_MODELS.Turbo0613 || model === OPENAI_MODELS.Turbo
+      ? modernJailbreak
+      : originalJailbreak
   return `
 Recent conversation history:
 {{history}}

common/dummy.ts

@@ -3,7 +3,10 @@ import { AppSchema } from './types/schema'
 
 let counter = Date.now()
 
-export function toChar(name: string, overrides?: Partial<AppSchema.Character>): AppSchema.Character {
+export function toChar(
+  name: string,
+  overrides?: Partial<AppSchema.Character>
+): AppSchema.Character {
   return {
     _id: name,
     kind: 'character',
@@ -130,7 +133,12 @@ export function toBook(name: string, entries: AppSchema.MemoryEntry[] = []): App
   }
 }
 
-export function toEntry(keywords: string[], entry: string, priority = 0, weight = 0): AppSchema.MemoryEntry {
+export function toEntry(
+  keywords: string[],
+  entry: string,
+  priority = 0,
+  weight = 0
+): AppSchema.MemoryEntry {
   return {
     enabled: true,
     name: entry,

common/guidance/guidance-parser.ts

@@ -166,7 +166,12 @@ function handlePipe(value: string, pipe?: Pipe) {
 
     case 'words': {
       const [first] = value.trim().split('\n')
-      const words = first.replace(/\n/g, ' ').split(' ').slice(0, pipe.value).join(' ').replace(/\./gi, '')
+      const words = first
+        .replace(/\n/g, ' ')
+        .split(' ')
+        .slice(0, pipe.value)
+        .join(' ')
+        .replace(/\./gi, '')
       return words
     }
   }

common/horde-gen.ts

@@ -22,7 +22,9 @@ type FetchOpts = {
   key?: string
 }
 
-type Fetcher = <T = any>(opts: FetchOpts) => Promise<{ statusCode?: number; statusMessage?: string; body: T }>
+type Fetcher = <T = any>(
+  opts: FetchOpts
+) => Promise<{ statusCode?: number; statusMessage?: string; body: T }>
 
 type HordeCheck = {
   generations: any[]
@@ -147,7 +149,10 @@ export async function generateText(
     top_k: preset.topK ?? defaultPresets.horde.topK,
     top_p: preset.topP ?? defaultPresets.horde.topP,
     typical: preset.typicalP ?? defaultPresets.horde.typicalP,
-    max_context_length: Math.min(preset.maxContextLength ?? defaultPresets.horde.maxContextLength, 2048),
+    max_context_length: Math.min(
+      preset.maxContextLength ?? defaultPresets.horde.maxContextLength,
+      2048
+    ),
     rep_pen: preset.repetitionPenalty ?? defaultPresets.horde.repetitionPenaltyRange,
     rep_pen_range: preset.repetitionPenaltyRange ?? defaultPresets.horde.repetitionPenaltyRange,
     rep_pen_slope: preset.repetitionPenaltySlope,
@@ -178,7 +183,9 @@ async function generate(opts: GenerateOpts) {
 
   if (init.statusCode && init.statusCode >= 400) {
     const error: any = new Error(
-      `Failed to create Horde generation ${init.statusCode} ${init.body.message || init.statusMessage}`
+      `Failed to create Horde generation ${init.statusCode} ${
+        init.body.message || init.statusMessage
+      }`
     )
     error.body = init.body
     throw error
@@ -213,7 +220,9 @@ async function poll(url: string, key: string | undefined, interval = 6.5) {
 
     const res = await useFetch<HordeCheck>({ method: 'get', url, key })
     if (res.statusCode && res.statusCode >= 400) {
-      const error: any = new Error(`Horde request failed (${res.statusCode}) ${res.body.message || res.statusMessage}`)
+      const error: any = new Error(
+        `Horde request failed (${res.statusCode}) ${res.body.message || res.statusMessage}`
+      )
       error.body = res.body
       throw error
     }

common/image-prompt.ts

@@ -13,7 +13,10 @@ export type ImagePromptOpts = {
 
 const appearanceKeys = ['appearance', 'looks']
 
-export async function createAppearancePrompt(user: AppSchema.User, avatar: { persona: AppSchema.Persona }) {
+export async function createAppearancePrompt(
+  user: AppSchema.User,
+  avatar: { persona: AppSchema.Persona }
+) {
   let visuals: string[] = []
 
   const max = getMaxImageContext(user)

common/image.ts

@@ -59,6 +59,9 @@ type ReverseMap<T extends Record<keyof T, keyof any>> = {
 }
 
 function reverseKeyValue<T extends { [key: string]: string }>(obj: T): ReverseMap<T> {
-  const reversed = Object.entries(obj).reduce((prev, [key, value]) => Object.assign(prev, { [value]: key }), {})
+  const reversed = Object.entries(obj).reduce(
+    (prev, [key, value]) => Object.assign(prev, { [value]: key }),
+    {}
+  )
   return reversed as any
 }

common/memory.ts

@@ -57,7 +57,9 @@ export type CharacterBook = {
     selective?: boolean /** if `true`, require a key from both `keys` and `secondary_keys` to trigger the entry */
     secondary_keys?: Array<string> /** see field `selective`. ignored if selective == false */
     constant?: boolean /** if true, always inserted in the prompt (within budget limit) */
-    position?: 'before_char' | 'after_char' /** whether the entry is placed before or after the character defs */
+    position?:
+      | 'before_char'
+      | 'after_char' /** whether the entry is placed before or after the character defs */
   }>
 }
 
@@ -149,7 +151,10 @@ export function buildMemoryPrompt(opts: MemoryOpts, encoder: Encoder): MemoryPro
   }
 }
 
-function byPriorityThenIndex({ id: lid, index: li, entry: l }: Match, { id: rid, index: ri, entry: r }: Match) {
+function byPriorityThenIndex(
+  { id: lid, index: li, entry: l }: Match,
+  { id: rid, index: ri, entry: r }: Match
+) {
   if (l.weight !== r.weight) return l.weight < r.weight ? 1 : -1
   if (li !== ri) return li > ri ? -1 : 1
   return lid > rid ? 1 : lid === rid ? 0 : -1

common/prompt.ts

@@ -1,6 +1,12 @@
 import type { GenerateRequestV2 } from '../srv/adapter/type'
 import type { AppSchema } from './types/schema'
-import { AIAdapter, NOVEL_MODELS, OPENAI_CHAT_MODELS, OPENAI_MODELS, SUPPORTS_INSTRUCT } from './adapters'
+import {
+  AIAdapter,
+  NOVEL_MODELS,
+  OPENAI_CHAT_MODELS,
+  OPENAI_MODELS,
+  SUPPORTS_INSTRUCT,
+} from './adapters'
 import { formatCharacter } from './characters'
 import { defaultTemplate } from './default-preset'
 import { IMAGE_SUMMARY_PROMPT } from './image'
@@ -182,7 +188,12 @@ export function createPrompt(opts: PromptOpts, encoder: Encoder, maxContext?: nu
  * @param lines Always in time-ascending order (oldest to newest)
  * @returns
  */
-export function createPromptWithParts(opts: GenerateRequestV2, parts: PromptParts, lines: string[], encoder: Encoder) {
+export function createPromptWithParts(
+  opts: GenerateRequestV2,
+  parts: PromptParts,
+  lines: string[],
+  encoder: Encoder
+) {
   const post = createPostPrompt(opts)
   const template = getTemplate(opts, parts)
   const history = { lines, order: 'asc' } as const
@@ -197,7 +208,10 @@ export function createPromptWithParts(opts: GenerateRequestV2, parts: PromptPart
   return { lines: history.lines, prompt, parts, post }
 }
 
-export function getTemplate(opts: Pick<GenerateRequestV2, 'settings' | 'chat'>, parts: PromptParts) {
+export function getTemplate(
+  opts: Pick<GenerateRequestV2, 'settings' | 'chat'>,
+  parts: PromptParts
+) {
   const isChat = OPENAI_CHAT_MODELS[opts.settings?.oaiModel || ''] ?? false
   const useGaslight = (opts.settings?.service === 'openai' && isChat) || opts.settings?.useGaslight
   const gaslight = opts.settings?.gaslight || defaultPresets.openai.gaslight
@@ -215,7 +229,10 @@ type InjectOpts = {
   encoder: Encoder
 }
 
-export function injectPlaceholders(template: string, { opts, parts, history: hist, encoder, ...rest }: InjectOpts) {
+export function injectPlaceholders(
+  template: string,
+  { opts, parts, history: hist, encoder, ...rest }: InjectOpts
+) {
   const profile = opts.members.find((mem) => mem.userId === opts.chat.userId)
   const sender = opts.impersonate?.name || profile?.handle || 'You'
 
@@ -225,7 +242,9 @@ export function injectPlaceholders(template: string, { opts, parts, history: his
     const next = hist.lines.filter((line) => !line.includes(SAMPLE_CHAT_MARKER))
 
     const postSample =
-      opts.settings?.service && SUPPORTS_INSTRUCT[opts.settings.service] ? SAMPLE_CHAT_MARKER : '<START>'
+      opts.settings?.service && SUPPORTS_INSTRUCT[opts.settings.service]
+        ? SAMPLE_CHAT_MARKER
+        : '<START>'
 
     const msg = `${SAMPLE_CHAT_PREAMBLE}\n${sampleChat}\n${postSample}`
       .replace(BOT_REPLACE, opts.replyAs.name)
@@ -421,7 +440,9 @@ export function getPromptParts(opts: PromptPartsOptions, lines: string[], encode
     if (!bot) continue
     if (personalities.has(bot._id)) continue
     personalities.add(bot._id)
-    parts.allPersonas.push(`${bot.name}'s personality: ${formatCharacter(bot.name, bot.persona, bot.persona.kind)}`)
+    parts.allPersonas.push(
+      `${bot.name}'s personality: ${formatCharacter(bot.name, bot.persona, bot.persona.kind)}`
+    )
   }
 
   if (chat.scenario && chat.overrides) {
@@ -434,7 +455,9 @@ export function getPromptParts(opts: PromptPartsOptions, lines: string[], encode
   }
 
   parts.sampleChat = (
-    replyAs._id === char._id && !!chat.overrides ? chat.sampleChat ?? replyAs.sampleChat : replyAs.sampleChat
+    replyAs._id === char._id && !!chat.overrides
+      ? chat.sampleChat ?? replyAs.sampleChat
+      : replyAs.sampleChat
   )
     .split('\n')
     .filter(removeEmpty)
@@ -513,7 +536,16 @@ function getSupplementaryParts(opts: PromptPartsOptions, replyAs: AppSchema.Char
 function createPostPrompt(
   opts: Pick<
     PromptOpts,
-    'kind' | 'chat' | 'char' | 'members' | 'continue' | 'settings' | 'user' | 'book' | 'replyAs' | 'impersonate'
+    | 'kind'
+    | 'chat'
+    | 'char'
+    | 'members'
+    | 'continue'
+    | 'settings'
+    | 'user'
+    | 'book'
+    | 'replyAs'
+    | 'impersonate'
   >
 ) {
   const post = []
@@ -578,7 +610,12 @@ function getLinesForPrompt(
   return lines
 }
 
-export function fillPromptWithLines(encoder: Encoder, tokenLimit: number, amble: string, lines: string[]) {
+export function fillPromptWithLines(
+  encoder: Encoder,
+  tokenLimit: number,
+  amble: string,
+  lines: string[]
+) {
   let count = encoder(amble)
   const adding: string[] = []
 
@@ -626,7 +663,8 @@ export function getChatPreset(
 
   // #1
   if (chat.genPreset) {
-    if (isDefaultPreset(chat.genPreset)) return { _id: chat.genPreset, ...defaultPresets[chat.genPreset] }
+    if (isDefaultPreset(chat.genPreset))
+      return { _id: chat.genPreset, ...defaultPresets[chat.genPreset] }
 
     const preset = userPresets.find((preset) => preset._id === chat.genPreset)
     if (preset) return preset
@@ -671,7 +709,8 @@ export function getAdapter(
   config: AppSchema.User,
   preset: Partial<AppSchema.GenSettings> | undefined
 ) {
-  const chatAdapter = !chat.adapter || chat.adapter === 'default' ? config.defaultAdapter : chat.adapter
+  const chatAdapter =
+    !chat.adapter || chat.adapter === 'default' ? config.defaultAdapter : chat.adapter
 
   let adapter = preset?.service ? preset.service : chatAdapter
 
@@ -719,8 +758,13 @@ export function getAdapter(
  * When we know the maximum context limit for a particular LLM, ensure that the context limit we use does not exceed it.
  */
 
-function getContextLimit(gen: Partial<AppSchema.GenSettings> | undefined, adapter: AIAdapter, model: string): number {
-  const configuredMax = gen?.maxContextLength || getFallbackPreset(adapter)?.maxContextLength || 2048
+function getContextLimit(
+  gen: Partial<AppSchema.GenSettings> | undefined,
+  adapter: AIAdapter,
+  model: string
+): number {
+  const configuredMax =
+    gen?.maxContextLength || getFallbackPreset(adapter)?.maxContextLength || 2048
 
   const genAmount = gen?.maxTokens || getFallbackPreset(adapter)?.maxTokens || 80
 

common/template-parser.ts

@@ -171,7 +171,9 @@ function renderIterator(holder: IterableHolder, children: CNode[], opts: ParseOp
   const output: string[] = []
 
   const entities =
-    holder === 'bots' ? Object.values(opts.characters).filter((b) => !!b && b._id !== opts.replyAs._id) : opts.lines
+    holder === 'bots'
+      ? Object.values(opts.characters).filter((b) => !!b && b._id !== opts.replyAs._id)
+      : opts.lines
 
   let i = 0
   for (const entity of entities) {
@@ -231,7 +233,11 @@ function getPlaceholder(node: PlaceHolder | ConditionNode, opts: ParseOpts) {
       return opts.replyAs.name
 
     case 'user':
-      return opts.impersonate?.name || opts.members.find((m) => m.userId === opts.user._id)?.handle || 'You'
+      return (
+        opts.impersonate?.name ||
+        opts.members.find((m) => m.userId === opts.user._id)?.handle ||
+        'You'
+      )
 
     case 'example_dialogue':
       return opts.parts.sampleChat?.join('\n') || ''