Fix model limits in sub page (#1037)

* Fix model limits in sub page * set chub flag to true on visit * fix menu close on chat tour start * first pass of chat completion api - simple preset mode - delete msg locally immediately - respond earlier when creating user message - use max context if known option - hide use recommended when simple - use recommended settings if available in simple mode - encourage user to change their display name - basic prompts: insert ujb in posts
agnaistic · Oct 2, 2024 · 88186e8 · 88186e8
1 parent 52e13ab
commit 88186e8
Show file tree

Hide file tree

Showing 56 changed files with 1,225 additions and 680 deletions.
diff --git a/common/adapters.ts b/common/adapters.ts
@@ -18,6 +18,12 @@ export type AdapterSetting = {
 
   setting: SettingType
   preset?: boolean
+
+  /**
+   * If enabled the setting won't be visible in "Simple" preset mode
+   * Defaults to true
+   */
+  advanced?: boolean
 }
 
 type SettingType =
@@ -33,6 +39,23 @@ export type AdapterOptions = {
   load?: (user?: AppSchema.User | null) => AdapterSetting[]
 }
 
+export const MODE_SETTINGS: {
+  [key in NonNullable<PresetAISettings['presetMode']>]?: {
+    [key in keyof AppSchema.GenSettings]?: boolean
+  }
+} = {
+  simple: {
+    maxContextLength: true,
+    maxTokens: true,
+    modelFormat: true,
+    ultimeJailbreak: true,
+    streamResponse: true,
+    temp: true,
+    localRequests: true,
+  },
+  advanced: {},
+}
+
 export const PERSONA_FORMATS = ['boostyle', 'wpp', 'sbf', 'attributes', 'text'] as const
 
 export const PERSONA_LABELS: { [key in PersonaFormat]: string } = {
@@ -393,8 +416,6 @@ export const settingLabels: { [key in keyof PresetAISettings]: string } = {
   addBosToken: 'Add BOS Token',
   antiBond: 'Anti-bond',
   banEosToken: 'Ban EOS Token',
-  cfgOppose: 'CFG Opposing Prompt',
-  cfgScale: 'CFG Scale',
   claudeModel: 'Claude Model',
   encoderRepitionPenalty: 'Encoder Repetition Penalty',
   frequencyPenalty: 'Frequency Penalty',

diff --git a/common/presets/agnaistic.ts b/common/presets/agnaistic.ts
@@ -6,7 +6,9 @@ export const agnaiPresets = {
   agnai: {
     service: 'agnaistic',
     name: 'Agnaistic',
+    presetMode: 'simple',
     maxTokens: 400,
+    useMaxContext: true,
     maxContextLength: 8192,
     repetitionPenalty: 1,
     repetitionPenaltySlope: 0,
@@ -16,7 +18,7 @@ export const agnaiPresets = {
     topP: 1,
     typicalP: 1,
     topA: 0,
-    minP: 0,
+    minP: 0.1,
     tailFreeSampling: 1,
     encoderRepitionPenalty: 1.0,
     penaltyAlpha: 0,
@@ -34,7 +36,6 @@ export const agnaiPresets = {
     memoryReverseWeight: false,
     antiBond: false,
     useAdvancedPrompt: 'basic',
-    promptOrderFormat: 'Alpaca',
     promptOrder: [
       {
         placeholder: 'system_prompt',

diff --git a/common/presets/kobold.ts b/common/presets/kobold.ts
@@ -5,8 +5,10 @@ export const koboldPresets = {
   basic: {
     service: 'kobold',
     name: 'Simple',
+    useMaxContext: true,
+    presetMode: 'simple',
     maxTokens: 300,
-    maxContextLength: 2048,
+    maxContextLength: 4096,
     repetitionPenalty: 1,
     repetitionPenaltySlope: 1,
     repetitionPenaltyRange: 1024,

diff --git a/common/presets/novel.ts b/common/presets/novel.ts
@@ -21,8 +21,6 @@ export const novelPresets = {
     topA: 0.1,
     order: [6, 2, 3, 0, 4, 1, 5],
     disabledSamplers: [6, 5],
-    cfgScale: 1,
-    cfgOppose: '',
     streamResponse: true,
     typicalP: 1,
     gaslight: `{{char}} Memory: {{memory}}

diff --git a/common/prompt-order.ts b/common/prompt-order.ts
@@ -28,44 +28,54 @@ export function promptOrderToTemplate(
 }
 
 function getOrderHolder(format: string, holder: string) {
-  return formatHolders[format]?.[holder] || fallbackHolders[holder] || ''
+  return formatHolders[format]?.[holder] || formatHolders.Universal[holder] || ''
 }
 
 export const formatHolders: Record<string, Record<string, string>> = {
+  Universal: {
+    system_prompt: neat`<system>{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}</system>`,
+    scenario: neat`{{#if scenario}}The scenario of the conversation:\n{{scenario}}\n{{/if}}`,
+    memory: neat`{{#if memory}}"{{char}}'s" memories:\n{{memory}}\n{{/if}}`,
+    personality: neat`{{#if personality}}{{char}}'s personality:\n{{personality}}\n{{/if}}`,
+    impersonating: neat`{{#if impersonating}}{{user}}'s personality:\n{{impersonating}}\n{{/if}}`,
+    chat_embed: neat`{{#if chat_embed}}Relevant past conversation history:\n{{chat_embed}}\n{{/if}}`,
+    example_dialogue: neat`{{#if example_dialogue}}How "{{char}}" speaks:\n{{example_dialogue}}\n{{/if}}`,
+    history: neat`Then the roleplay chat between {{user}} and {{char}} begins.
+    
+    {{#each msg}}{{#if .isbot}}<bot>{{/if}}{{#if .isuser}}<user>{{/if}}{{.name}}: {{.msg}}{{#if .isbot}}</bot>{{/if}}{{#if .isuser}}</user>{{/if}}{{/each}}`,
+    post: neat`<bot>{{#if ujb}}({{value}}) {{/if}}{{post}}`,
+  },
   Alpaca: {
     system_prompt: `### System:\n{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}`,
     history: neat`Then the roleplay chat between "{{user}}" and "{{char}}" begins.
     
     {{#each msg}}{{#if .isbot}}### Response:\n{{.name}}: {{.msg}}{{/if}}{{#if .isuser}}### Instruction:\n{{.name}}: {{.msg}}{{/if}}
     {{/each}}`,
-    post: `### Response:\n{{post}}`,
-    ujb: `{{#if ujb}}({{ujb}}){{/if}}`,
+    post: `### Response:\n{{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
   Vicuna: {
     system_prompt: neat`{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}`,
     history: neat`Then the roleplay chat between "{{user}}" and "{{char}}" begins.
     
     {{#each msg}}{{#if .isbot}}ASSISTANT:\n{{.name}}: {{.msg}}{{/if}}{{#if .isuser}}USER:\n{{.name}}: {{.msg}}{{/if}}
     {{/each}}`,
-    post: `ASSISTANT: {{post}}`,
-    ujb: `{{#if ujb}}({{ujb}}){{/if}}`,
+    post: `ASSISTANT: {{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
   Mistral: {
     system_prompt: neat`[INST] {{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}} [/INST]`,
     history: neat`Then the roleplay chat between "{{user}}" and "{{char}}" begins.
     
     {{#each msg}}{{#if .isbot}}\n{{.name}}: {{.msg}}{{/if}}{{#if .isuser}}[INST] {{.name}}: {{.msg}} [/INST]{{/if}}
     {{/each}}`,
-    post: `{{post}}`,
-    ujb: `{{#if ujb}}[INST] {{ujb}} [/INST]{{/if}}`,
+    post: `{{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
   Metharme: {
     system_prompt: `<|system|>{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}`,
     history: neat`Then the roleplay chat between "{{user}}" and "{{char}}" begins.
     
     {{#each msg}}{{#if .isbot}}<|model|>{{/if}}{{#if .isuser}}<|user|>{{/if}}{{.name}}: {{.msg}}
     {{/each}}`,
-    post: `<|model|>{{post}}`,
+    post: `<|model|>{{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
   ChatML: {
     system_prompt: neat`<|im_start|>system
@@ -78,12 +88,9 @@ export const formatHolders: Record<string, Record<string, string>> = {
     example_dialogue: neat`{{#if example_dialogue}}How "{{char}}" speaks:\n{{example_dialogue}}\n{{/if}}`,
     history: neat`Then the roleplay chat between {{user}} and {{char}} begins.
     
-    {{#each msg}}<|im_start|>{{#if .isbot}}assistant{{/if}}{{#if .isuser}}user{{/if}}
-    {{.name}}: {{.msg}}<|im_end|>{{/each}}`,
+    {{#each msg}}<|im_start|>{{#if .isbot}}assistant{{/if}}{{#if .isuser}}user{{/if}}{{.name}}: {{.msg}}<|im_end|>{{/each}}`,
     post: neat`<|im_start|>assistant
-    {{post}}`,
-    ujb: neat`{{#if ujb}}<|im_start|>system
-    {{ujb}}<|im_end|>{{/if}}`,
+    {{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
   Llama3: {
     system_prompt: neat`<|begin_of_text|><|start_header_id|>system
@@ -96,29 +103,26 @@ export const formatHolders: Record<string, Record<string, string>> = {
     example_dialogue: neat`{{#if example_dialogue}}How "{{char}}" speaks:\n{{example_dialogue}}\n{{/if}}`,
     history: neat`Then the roleplay chat between "{{user}}" and "{{char}}" begins.
     
-    {{#each msg}}<|start_header_id|>{{#if .isbot}}assistant{{/if}}{{#if .isuser}}user{{/if}}
-    {{.name}}: {{.msg}}<|eot_id|>{{/each}}`,
+    {{#each msg}}<|start_header_id|>{{#if .isbot}}assistant{{/if}}{{#if .isuser}}user{{/if}}{{.name}}: {{.msg}}<|eot_id|>{{/each}}`,
     post: neat`<|start_header_id|>assistant
-    {{post}}`,
-    ujb: neat`{{#if ujb}}<|im_start|>system
-    {{ujb}}<|im_end|>{{/if}}`,
+    {{#if ujb}}({{value}}) {{/if}}{{post}}`,
   },
-  'Pyg/Simple:': {
+  'Pyg/Simple': {
     history: `Start of the conversation:\n\n{{history}}`,
     scenario: `{{#if scenario}}Scenario: {{scenario}}{{/if}}`,
   },
 }
 
-const fallbackHolders: Record<string, string> = {
-  system_prompt: neat`{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}`,
-  scenario: `{{#if scenario}}The scenario of the conversation:\n{{scenario}}\n{{/if}}`,
-  personality: `{{#if personality}}"{{char}}'s" personality:\n{{personality}}\n{{/if}}`,
-  memory: `{{#if memory}}"{{char}}'s" memories:\n{{memory}}\n{{/if}}`,
-  ujb: `{{#if ujb}}({{ujb}}) {{/if}}`,
-  example_dialogue: `{{#if example_dialogue}}How "{{char}}" speaks:\n{{example_dialogue}}\n{{/if}}`,
-  impersonating: `{{#if impersonating}}"{{user}}'s" personality:\n{{impersonating}}\n{{/if}}`,
-  history: `{{history}}`,
-  post: `{{post}}`,
-  chat_embed: `{{#if chat_embed}}Relevant past conversation history:\n{{chat_embed}}\n{{/if}}`,
-  user_embed: `{{#if user_embed}}Relevant information to the conversation:\n{{user_embed}}\n{{/if}}`,
-}
+// const fallbackHolders: Record<string, string> = {
+//   system_prompt: neat`{{#if system_prompt}}{{value}}{{#else}}Write "{{char}}'s" next reply in a fictional roleplay chat between "{{char}}" and "{{user}}"{{/else}}{{/if}}`,
+//   scenario: `{{#if scenario}}The scenario of the conversation:\n{{scenario}}\n{{/if}}`,
+//   personality: `{{#if personality}}"{{char}}'s" personality:\n{{personality}}\n{{/if}}`,
+//   memory: `{{#if memory}}"{{char}}'s" memories:\n{{memory}}\n{{/if}}`,
+//   ujb: `{{#if ujb}}({{ujb}}) {{/if}}`,
+//   example_dialogue: `{{#if example_dialogue}}How "{{char}}" speaks:\n{{example_dialogue}}\n{{/if}}`,
+//   impersonating: `{{#if impersonating}}"{{user}}'s" personality:\n{{impersonating}}\n{{/if}}`,
+//   history: `{{history}}`,
+//   post: `{{post}}`,
+//   chat_embed: `{{#if chat_embed}}Relevant past conversation history:\n{{chat_embed}}\n{{/if}}`,
+//   user_embed: `{{#if user_embed}}Relevant information to the conversation:\n{{user_embed}}\n{{/if}}`,
+// }
diff --git a/common/prompt.ts b/common/prompt.ts
@@ -239,7 +239,7 @@ export async function createPromptParts(opts: PromptOpts, encoder: TokenCounter)
    * The queryable embeddings are messages that are _NOT_ included in the context
    */
   const maxContext = opts.settings
-    ? opts.settings.maxContextLength! - templateSize - opts.settings.maxTokens!
+    ? getContextLimit(opts.user, opts.settings) - templateSize - opts.settings.maxTokens!
     : undefined
   const lines = await getLinesForPrompt(opts, encoder, maxContext)
   const parts = await buildPromptParts(opts, lines, encoder)
@@ -290,16 +290,16 @@ export async function assemblePrompt(
 
 export function getTemplate(opts: Pick<GenerateRequestV2, 'settings' | 'chat'>) {
   const fallback = getFallbackPreset(opts.settings?.service!)
-  if (
-    opts.settings?.useAdvancedPrompt === 'basic' &&
-    opts.settings.promptOrderFormat &&
-    opts.settings.promptOrder
-  ) {
-    const template = promptOrderToTemplate(
-      opts.settings.promptOrderFormat,
-      opts.settings.promptOrder
-    )
-    return template
+  if (opts.settings?.useAdvancedPrompt === 'basic' || opts.settings?.presetMode === 'simple') {
+    if (opts.settings.presetMode === 'simple') {
+      const template = promptOrderToTemplate('Universal', simpleOrder)
+      return template
+    }
+
+    if (opts.settings.modelFormat && opts.settings.promptOrder) {
+      const template = promptOrderToTemplate(opts.settings.modelFormat, opts.settings.promptOrder)
+      return template
+    }
   }
 
   const template = opts.settings?.gaslight || fallback?.gaslight || defaultTemplate
@@ -322,6 +322,17 @@ type InjectOpts = {
   encoder: TokenCounter
 }
 
+const simpleOrder: NonNullable<AppSchema.GenSettings['promptOrder']> = [
+  'system_prompt',
+  'scenario',
+  'personality',
+  'impersonating',
+  'chat_embed',
+  'memory',
+  'example_dialogue',
+  'history',
+].map((placeholder) => ({ placeholder, enabled: true }))
+
 export async function injectPlaceholders(template: string, inject: InjectOpts) {
   const { opts, parts, history: hist, encoder, ...rest } = inject
 
@@ -352,8 +363,6 @@ export async function injectPlaceholders(template: string, inject: InjectOpts) {
     hist.lines = next
   }
 
-  const { adapter, model } = getAdapter(opts.chat, opts.user, opts.settings)
-
   const lines = !hist
     ? []
     : hist.order === 'desc'
@@ -368,7 +377,7 @@ export async function injectPlaceholders(template: string, inject: InjectOpts) {
     lines,
     ...rest,
     limit: {
-      context: getContextLimit(opts.user, opts.settings, adapter, model),
+      context: getContextLimit(opts.user, opts.settings),
       encoder,
     },
   })
@@ -609,8 +618,7 @@ export async function getLinesForPrompt(
   encoder: TokenCounter,
   maxContext?: number
 ) {
-  const { adapter, model } = getAdapter(opts.chat, opts.user, settings)
-  maxContext = maxContext || getContextLimit(opts.user, settings, adapter, model)
+  maxContext = maxContext || getContextLimit(opts.user, settings)
 
   const profiles = new Map<string, AppSchema.Profile>()
   for (const member of members) {
@@ -847,7 +855,7 @@ export function getAdapter(
     }
   }
 
-  const contextLimit = getContextLimit(user, preset, adapter, model)
+  const contextLimit = getContextLimit(user, preset)
 
   return { adapter, model, preset: presetName, contextLimit, isThirdParty }
 }
@@ -868,31 +876,34 @@ export function setContextLimitStrategy(strategy: LimitStrategy) {
 
 export function getContextLimit(
   user: AppSchema.User,
-  gen: Partial<AppSchema.GenSettings> | undefined,
-  adapter: AIAdapter,
-  model: string
+  gen: Partial<AppSchema.GenSettings> | undefined
 ): number {
-  const genAmount = gen?.maxTokens || getFallbackPreset(adapter)?.maxTokens || 80
+  const genAmount = gen?.maxTokens || getFallbackPreset(gen?.service || 'horde')?.maxTokens || 80
   const configuredMax =
-    gen?.maxContextLength || getFallbackPreset(adapter)?.maxContextLength || 2048
+    gen?.maxContextLength || getFallbackPreset(gen?.service || 'horde')?.maxContextLength || 4096
 
-  if (gen?.service === 'kobold' || gen?.service === 'ooba') return configuredMax - genAmount
+  if (!gen?.service) return configuredMax - genAmount
 
-  switch (adapter) {
+  switch (gen.service) {
     case 'agnaistic': {
       const stratMax = _strategy(user, gen)
+      if (gen?.useMaxContext && stratMax) {
+        return stratMax.context - genAmount
+      }
+
       const max = Math.min(configuredMax, stratMax?.context ?? configuredMax)
       return max - genAmount
     }
 
     // Any LLM could be used here so don't max any assumptions
+    case 'ooba':
     case 'petals':
     case 'kobold':
     case 'horde':
-    case 'ooba':
       return configuredMax - genAmount
 
     case 'novel': {
+      const model = gen?.novelModel || NOVEL_MODELS.kayra_v1
       if (model === NOVEL_MODELS.clio_v1 || model === NOVEL_MODELS.kayra_v1) {
         return Math.min(8000, configuredMax) - genAmount
       }
@@ -901,6 +912,7 @@ export function getContextLimit(
     }
 
     case 'openai': {
+      const model = (gen?.service === 'openai' ? gen?.oaiModel! : gen?.thirdPartyModel) || ''
       const limit = OPENAI_CONTEXTS[model] || 128000
       return Math.min(configuredMax, limit) - genAmount
     }
@@ -919,6 +931,8 @@ export function getContextLimit(
 
     case 'openrouter':
       if (gen?.openRouterModel) {
+        if (gen.useMaxContext) return gen.openRouterModel.context_length - genAmount
+
         return Math.min(gen.openRouterModel.context_length, configuredMax) - genAmount
       }
 

diff --git a/common/requests/payloads.ts b/common/requests/payloads.ts
@@ -338,7 +338,6 @@ function getBasePayload(opts: PayloadOpts, stops: string[] = []) {
       dynatemp_range: gen.dynatemp_range,
       dynatemp_exponent: gen.dynatemp_exponent,
       smoothing_factor: gen.smoothingFactor,
-      trim_stop: gen.trimStop,
       rep_pen_range: gen.repetitionPenaltyRange,
       rep_pen_slope: gen.repetitionPenaltySlope,
     }

diff --git a/common/types/presets.ts b/common/types/presets.ts
@@ -76,6 +76,7 @@ export interface UserGenPreset extends GenSettings {
 export interface GenSettings {
   name: string
   description?: string
+  presetMode?: 'simple' | 'advanced'
 
   service?: AIAdapter
 
@@ -87,6 +88,7 @@ export interface GenSettings {
   smoothingCurve?: number
   maxTokens: number
   maxContextLength?: number
+  useMaxContext?: boolean
   repetitionPenalty: number
   repetitionPenaltyRange: number
   repetitionPenaltySlope: number