From 0c60707fe4715584d86b648dc0d5b1b114b1ad8e Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Mon, 18 Nov 2024 11:17:40 +0000 Subject: [PATCH 01/14] support anthropic PDF beta --- package-lock.json | 8 ++-- package.json | 2 +- .../endpoints/anthropic/endpointAnthropic.ts | 6 +++ src/lib/server/endpoints/anthropic/utils.ts | 43 +++++++++++++++++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 50fb77ce552..39925a1cde0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -101,7 +101,7 @@ "vitest": "^2.1.4" }, "optionalDependencies": { - "@anthropic-ai/sdk": "^0.25.0", + "@anthropic-ai/sdk": "^0.32.1", "@anthropic-ai/vertex-sdk": "^0.4.1", "@aws-sdk/client-bedrock-runtime": "^3.631.0", "@google-cloud/vertexai": "^1.1.0", @@ -265,9 +265,9 @@ } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.25.2", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.25.2.tgz", - "integrity": "sha512-F1Hck/asswwidFLtGdMg3XYgRxEUfygNbpkq5KEaEGsHNaSfxeX18/uZGQCL0oQNcj/tYNx8BaFXVwRhFDi45g==", + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", + "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", "optional": true, "dependencies": { "@types/node": "^18.11.18", diff --git a/package.json b/package.json index 24478306771..48f01718a46 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "zod": "^3.22.3" }, "optionalDependencies": { - "@anthropic-ai/sdk": "^0.25.0", + "@anthropic-ai/sdk": "^0.32.1", "@anthropic-ai/vertex-sdk": "^0.4.1", "@aws-sdk/client-bedrock-runtime": "^3.631.0", "@google-cloud/vertexai": "^1.1.0", diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 889bceeff0c..a1d542f4c91 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -4,6 +4,7 @@ import { env } from "$env/dynamic/private"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { createImageProcessorOptionsValidator } from "../images"; import { endpointMessagesToAnthropicMessages } from "./utils"; +import { createDocumentProcessorOptionsValidator } from "../document"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -23,6 +24,10 @@ export const endpointAnthropicParametersSchema = z.object({ maxWidth: 4096, maxHeight: 4096, }), + document: createDocumentProcessorOptionsValidator({ + supportedMimeTypes: ["application/pdf"], + maxSizeInMB: 32, + }), }) .default({}), }); @@ -39,6 +44,7 @@ export async function endpointAnthropic( throw new Error("Failed to import @anthropic-ai/sdk", { cause: e }); } + // console.log("---->" + JSON.stringify(mime)); const anthropic = new Anthropic({ apiKey, baseURL, diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index 91dd6929a39..0e85fff5c1d 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -1,8 +1,18 @@ import { makeImageProcessor, type ImageProcessorOptions } from "../images"; +import { makeDocumentProcessor, type FileProcessorOptions } from "../document"; import type { EndpointMessage } from "../endpoints"; import type { MessageFile } from "$lib/types/Message"; import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; +export interface DocumentBlockParam { + type: "document"; + source: { + data: string; + media_type: "application/pdf"; + type: "base64"; + }; +} + export async function fileToImageBlock( file: MessageFile, opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> @@ -20,11 +30,30 @@ export async function fileToImageBlock( }; } -type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" }; +export async function fileToDocumentBlock( + file: MessageFile, + opts: FileProcessorOptions<"application/pdf"> +): Promise { + const processor = makeDocumentProcessor(opts); + const { file: document, mime } = processor(file); + + return { + type: "document", + source: { + type: "base64", + media_type: mime, + data: document.toString("base64"), + }, + }; +} +type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" }; export async function endpointMessagesToAnthropicMessages( messages: EndpointMessage[], - multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> } + multimodal: { + image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">; + document: ImageProcessorOptions<"application/pdf">; + } ): Promise { return await Promise.all( messages @@ -34,7 +63,15 @@ export async function endpointMessagesToAnthropicMessages( role: message.from, content: [ ...(await Promise.all( - (message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image)) + (message.files ?? []).map(async (file) => { + if (file.mime.startsWith("image/")) { + return fileToImageBlock(file, multimodal.image); + } else if (file.mime === "application/pdf") { + return fileToDocumentBlock(file, multimodal.document); + } else { + throw new Error("Unsupported file type"); + } + }) )), { type: "text", text: message.content }, ], From 84aac49d4c7bb4a1d5c8c9409c588d695e6aef60 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:17:18 +0000 Subject: [PATCH 02/14] upstream merge, remove commented out console log line --- src/lib/server/endpoints/anthropic/endpointAnthropic.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index a1d542f4c91..1428990e6a5 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -44,7 +44,6 @@ export async function endpointAnthropic( throw new Error("Failed to import @anthropic-ai/sdk", { cause: e }); } - // console.log("---->" + JSON.stringify(mime)); const anthropic = new Anthropic({ apiKey, baseURL, From 92bf923151823c6bb93d941dbba43e06d1c57c79 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:58:05 +0000 Subject: [PATCH 03/14] Fixing type errors. the anthropic API does not yet include a "DocumentBlock" for support PDFs, so an extended type has been added to the endpoint. --- .../endpoints/anthropic/endpointAnthropic.ts | 6 ++++- .../anthropic/endpointAnthropicVertex.ts | 6 ++++- src/lib/server/endpoints/anthropic/utils.ts | 22 ++++++++++++++----- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 1428990e6a5..50b422f9500 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -5,6 +5,7 @@ import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { createImageProcessorOptionsValidator } from "../images"; import { endpointMessagesToAnthropicMessages } from "./utils"; import { createDocumentProcessorOptionsValidator } from "../document"; +import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -64,7 +65,10 @@ export async function endpointAnthropic( return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: await endpointMessagesToAnthropicMessages(messages, multimodal), + messages: (await endpointMessagesToAnthropicMessages( + messages, + multimodal + )) as MessageParam[], max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, top_p: parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts index a90dd627b21..06ceae7463c 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts @@ -3,6 +3,7 @@ import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { createImageProcessorOptionsValidator } from "../images"; import { endpointMessagesToAnthropicMessages } from "./utils"; +import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; export const endpointAnthropicVertexParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -56,7 +57,10 @@ export async function endpointAnthropicVertex( return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: await endpointMessagesToAnthropicMessages(messages, multimodal), + messages: (await endpointMessagesToAnthropicMessages( + messages, + multimodal + )) as MessageParam[], max_tokens: model.parameters?.max_new_tokens, temperature: model.parameters?.temperature, top_p: model.parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index 0e85fff5c1d..51ee573de1e 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -2,7 +2,11 @@ import { makeImageProcessor, type ImageProcessorOptions } from "../images"; import { makeDocumentProcessor, type FileProcessorOptions } from "../document"; import type { EndpointMessage } from "../endpoints"; import type { MessageFile } from "$lib/types/Message"; -import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; +import type { + ImageBlockParam, + MessageParam, + TextBlockParam, +} from "@anthropic-ai/sdk/resources/messages.mjs"; export interface DocumentBlockParam { type: "document"; @@ -13,6 +17,12 @@ export interface DocumentBlockParam { }; } +export type ExtendedContentBlock = ImageBlockParam | TextBlockParam | DocumentBlockParam; + +export interface ExtendedMessageParam extends Omit { + content: ExtendedContentBlock[]; +} + export async function fileToImageBlock( file: MessageFile, opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> @@ -52,13 +62,13 @@ export async function endpointMessagesToAnthropicMessages( messages: EndpointMessage[], multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">; - document: ImageProcessorOptions<"application/pdf">; + document?: FileProcessorOptions<"application/pdf">; // Make optional } -): Promise { +): Promise { return await Promise.all( messages .filter((message): message is NonSystemMessage => message.from !== "system") - .map>(async (message) => { + .map>(async (message) => { return { role: message.from, content: [ @@ -66,10 +76,10 @@ export async function endpointMessagesToAnthropicMessages( (message.files ?? []).map(async (file) => { if (file.mime.startsWith("image/")) { return fileToImageBlock(file, multimodal.image); - } else if (file.mime === "application/pdf") { + } else if (file.mime === "application/pdf" && multimodal.document) { return fileToDocumentBlock(file, multimodal.document); } else { - throw new Error("Unsupported file type"); + throw new Error(`Unsupported file type: ${file.mime}`); } }) )), From 3e34b63cccc3bd1c914c517345ed9f38f0ef8edc Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Mon, 18 Nov 2024 21:18:33 +0000 Subject: [PATCH 04/14] changed document processor to async (matching image processor) --- src/lib/server/endpoints/anthropic/utils.ts | 2 +- src/lib/server/endpoints/document.ts | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index 51ee573de1e..ef2f7666200 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -45,7 +45,7 @@ export async function fileToDocumentBlock( opts: FileProcessorOptions<"application/pdf"> ): Promise { const processor = makeDocumentProcessor(opts); - const { file: document, mime } = processor(file); + const { file: document, mime } = await processor(file); return { type: "document", diff --git a/src/lib/server/endpoints/document.ts b/src/lib/server/endpoints/document.ts index 5538e7b2735..70df7c09279 100644 --- a/src/lib/server/endpoints/document.ts +++ b/src/lib/server/endpoints/document.ts @@ -34,15 +34,21 @@ export type DocumentProcessor = (file: Messag mime: TMimeType; }; +export type AsyncDocumentProcessor = ( + file: MessageFile +) => Promise<{ + file: Buffer; + mime: TMimeType; +}>; + export function makeDocumentProcessor( options: FileProcessorOptions -): DocumentProcessor { - return (file) => { +): AsyncDocumentProcessor { + return async (file) => { const { supportedMimeTypes, maxSizeInMB } = options; const { mime, value } = file; const buffer = Buffer.from(value, "base64"); - const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000; if (tooLargeInBytes) { @@ -50,7 +56,6 @@ export function makeDocumentProcessor( } const outputMime = validateMimeType(supportedMimeTypes, mime); - return { file: buffer, mime: outputMime }; }; } From 36a1cc368cf3bb4ccd32f2794d5c13a56815df90 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:12:23 +0000 Subject: [PATCH 05/14] use the beta api types rather than custom extension --- src/lib/server/endpoints/anthropic/utils.ts | 33 ++++++--------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index ef2f7666200..0239e426300 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -3,30 +3,15 @@ import { makeDocumentProcessor, type FileProcessorOptions } from "../document"; import type { EndpointMessage } from "../endpoints"; import type { MessageFile } from "$lib/types/Message"; import type { - ImageBlockParam, - MessageParam, - TextBlockParam, -} from "@anthropic-ai/sdk/resources/messages.mjs"; - -export interface DocumentBlockParam { - type: "document"; - source: { - data: string; - media_type: "application/pdf"; - type: "base64"; - }; -} - -export type ExtendedContentBlock = ImageBlockParam | TextBlockParam | DocumentBlockParam; - -export interface ExtendedMessageParam extends Omit { - content: ExtendedContentBlock[]; -} + BetaImageBlockParam, + BetaMessageParam, + BetaBase64PDFBlock, +} from "@anthropic-ai/sdk/resources/beta/messages/messages.mjs"; export async function fileToImageBlock( file: MessageFile, opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> -): Promise { +): Promise { const processor = makeImageProcessor(opts); const { image, mime } = await processor(file); @@ -43,7 +28,7 @@ export async function fileToImageBlock( export async function fileToDocumentBlock( file: MessageFile, opts: FileProcessorOptions<"application/pdf"> -): Promise { +): Promise { const processor = makeDocumentProcessor(opts); const { file: document, mime } = await processor(file); @@ -62,13 +47,13 @@ export async function endpointMessagesToAnthropicMessages( messages: EndpointMessage[], multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">; - document?: FileProcessorOptions<"application/pdf">; // Make optional + document?: FileProcessorOptions<"application/pdf">; } -): Promise { +): Promise { return await Promise.all( messages .filter((message): message is NonSystemMessage => message.from !== "system") - .map>(async (message) => { + .map>(async (message) => { return { role: message.from, content: [ From 786b57637001041021fe39fe76f3d636e3492743 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:15:44 +0000 Subject: [PATCH 06/14] rudimentary tool testing --- .../endpoints/anthropic/endpointAnthropic.ts | 238 ++++++++++++++++-- 1 file changed, 224 insertions(+), 14 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 889bceeff0c..d0b31b2fa40 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -4,6 +4,17 @@ import { env } from "$env/dynamic/private"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { createImageProcessorOptionsValidator } from "../images"; import { endpointMessagesToAnthropicMessages } from "./utils"; +import type { + Tool, + ToolCall, + ToolInput, + ToolInputFile, + ToolInputFixed, + ToolInputOptional, + ToolResult, +} from "$lib/types/Tool"; +import type Anthropic from "@anthropic-ai/sdk"; +import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -46,7 +57,12 @@ export async function endpointAnthropic( defaultQuery, }); - return async ({ messages, preprompt, generateSettings }) => { + // get tool results and tools from this call + // convert from library to anthropic format of tools + // check if model has tools enabled + // process toolresults and add them to the request + // add a toolcall yield if the model stops for tool calling reasons. + return async ({ messages, preprompt, generateSettings, tools, toolResults }) => { let system = preprompt; if (messages?.[0]?.from === "system") { system = messages[0].content; @@ -56,10 +72,74 @@ export async function endpointAnthropic( const parameters = { ...model.parameters, ...generateSettings }; + let anthropicTools: Anthropic.Messages.Tool[] = []; + + if (model.tools && tools && tools.length > 0) { + anthropicTools = createAnthropicTools(tools); + } + anthropicTools.reverse().shift(); + console.log(JSON.stringify(anthropicTools)); + // Example tool for weather + const weatherTool: Anthropic.Messages.Tool[] = [ + { + name: "get_weather", + description: "Get the current weather in a given location", + input_schema: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + description: "The unit of temperature, either 'celsius' or 'fahrenheit'", + }, + }, + required: ["location"], + }, + }, + ]; + + const calculatorTool: Anthropic.Messages.Tool[] = [ + { + name: "calculator", + description: "Calculate the result of a mathematical expression", + input_schema: { + type: "object", + properties: { + equation: { + description: + "A mathematical expression to be evaluated. The result of the expression will be returned.", + type: "string", + }, + }, + required: ["equation"], + }, + }, + ]; + + console.log("-------------"); + console.log(JSON.stringify(weatherTool)); + /* if (!model.tools) { + anthropicTools = weatherTool; + } + +*/ + + let anthropic_messages = await endpointMessagesToAnthropicMessages(messages, multimodal); + if (toolResults && toolResults.length > 0) { + anthropic_messages = addToolResults(anthropic_messages, toolResults); + } + + // console.log(JSON.stringify(anthropic_messages)); return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: await endpointMessagesToAnthropicMessages(messages, multimodal), + tools: calculatorTool, + tool_choice: { type: "auto", disable_parallel_tool_use: false }, + messages: anthropic_messages, max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, top_p: parameters?.top_p, @@ -68,24 +148,59 @@ export async function endpointAnthropic( system, }); while (true) { - const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]); + const result = await Promise.race([ + // stream.emitted("inputJson"), + stream.emitted("text"), + stream.emitted("end"), + ]); // Stream end if (result === undefined) { - yield { - token: { - id: tokenId++, - text: "", - logprob: 0, - special: true, - }, - generated_text: await stream.finalText(), - details: null, - } satisfies TextGenerationStreamOutput; + stream.receivedMessages.forEach((message) => { + console.log("--->" + message.id + "..." + message.stop_reason + ".." + message.type); + message.content.forEach((contentBlock) => { + if (contentBlock.type === "tool_use") { + console.log( + "Tool call:", + contentBlock.id + contentBlock.name + JSON.stringify(contentBlock.input) + ); + } + }); + }); + + if ("tool_use" === stream.receivedMessages[0].stop_reason) { + const toolCalls: ToolCall[] = stream.receivedMessages[0].content + .filter( + (block): block is Anthropic.Messages.ContentBlock & { type: "tool_use" } => + block.type === "tool_use" + ) + .map((block) => ({ + name: block.name, + parameters: block.input as Record, + id: block.id, + })); + + yield { + token: { id: tokenId, text: "", logprob: 0, special: false, toolCalls }, + generated_text: null, + details: null, + }; + } else { + yield { + token: { + id: tokenId++, + text: "", + logprob: 0, + special: true, + }, + generated_text: await stream.finalText(), + details: null, + } satisfies TextGenerationStreamOutput; + } + return; } - // Text delta yield { token: { id: tokenId++, @@ -100,3 +215,98 @@ export async function endpointAnthropic( })(); }; } + +function addToolResults(messages: MessageParam[], toolResults: ToolResult[]): MessageParam[] { + const assistantMessages: MessageParam[] = []; + const userMessages: MessageParam[] = []; + + assistantMessages.push({ + role: "assistant", + content: [ + { + type: "tool_use", + id: "any_id", + name: toolResults[0].call.name, + input: toolResults[0].call.parameters, // Changed from JSON.stringify to direct object + }, + ], + }); + + userMessages.push({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "any_id", + is_error: toolResults[0].status === "error", + content: + toolResults[0].status === "error" + ? JSON.stringify(toolResults[0].message) // Include error message if it's an error + : JSON.stringify("outputs" in toolResults[0] ? toolResults[0].outputs : ""), // Otherwise include the output + }, + ], + }); + + return [...messages, ...assistantMessages, ...userMessages]; +} + +function createAnthropicTools(tools: Tool[]): Anthropic.Messages.Tool[] { + return tools.map((tool) => { + const properties = tool.inputs.reduce((acc, input) => { + acc[input.name] = convertToolInputToJSONSchema(input); + return acc; + }, {} as Record); + + const required = tool.inputs + .filter((input) => input.paramType === "required") + .map((input) => input.name); + + return { + name: tool.name, + description: tool.description, + input_schema: { + type: "object", + properties, + required: required.length > 0 ? required : undefined, + }, + }; + }); +} + +function convertToolInputToJSONSchema(input: ToolInput): Record { + const baseSchema: Record = {}; + if ("description" in input) { + baseSchema["description"] = input.description || ""; + } + switch (input.paramType) { + case "optional": + baseSchema["default"] = (input as ToolInputOptional).default; + break; + case "fixed": + baseSchema["const"] = (input as ToolInputFixed).value; + break; + } + + if (input.type === "file") { + baseSchema["type"] = "string"; + baseSchema["format"] = "binary"; + baseSchema["mimeTypes"] = (input as ToolInputFile).mimeTypes; + } else { + switch (input.type) { + case "str": + baseSchema["type"] = "string"; + break; + case "int": + baseSchema["type"] = "integer"; + break; + case "float": + baseSchema["type"] = "number"; + break; + case "bool": + baseSchema["type"] = "boolean"; + break; + } + } + + return baseSchema; +} From da224024ce7f94f9ba2def6acb09d755da0f1ffe Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 08:01:20 +0000 Subject: [PATCH 07/14] interim commit (tool re-passing, file handling) --- .../endpoints/anthropic/endpointAnthropic.ts | 119 +++++------------- src/lib/server/textGeneration/generate.ts | 4 +- src/lib/server/textGeneration/index.ts | 7 +- 3 files changed, 43 insertions(+), 87 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index d0b31b2fa40..656f45aa445 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -77,57 +77,7 @@ export async function endpointAnthropic( if (model.tools && tools && tools.length > 0) { anthropicTools = createAnthropicTools(tools); } - anthropicTools.reverse().shift(); - console.log(JSON.stringify(anthropicTools)); - // Example tool for weather - const weatherTool: Anthropic.Messages.Tool[] = [ - { - name: "get_weather", - description: "Get the current weather in a given location", - input_schema: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], - description: "The unit of temperature, either 'celsius' or 'fahrenheit'", - }, - }, - required: ["location"], - }, - }, - ]; - - const calculatorTool: Anthropic.Messages.Tool[] = [ - { - name: "calculator", - description: "Calculate the result of a mathematical expression", - input_schema: { - type: "object", - properties: { - equation: { - description: - "A mathematical expression to be evaluated. The result of the expression will be returned.", - type: "string", - }, - }, - required: ["equation"], - }, - }, - ]; - - console.log("-------------"); - console.log(JSON.stringify(weatherTool)); - /* if (!model.tools) { - anthropicTools = weatherTool; - } - -*/ - + console.log(JSON.stringify(messages)); let anthropic_messages = await endpointMessagesToAnthropicMessages(messages, multimodal); if (toolResults && toolResults.length > 0) { anthropic_messages = addToolResults(anthropic_messages, toolResults); @@ -137,8 +87,9 @@ export async function endpointAnthropic( return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - tools: calculatorTool, - tool_choice: { type: "auto", disable_parallel_tool_use: false }, + tools: anthropicTools, + tool_choice: + tools?.length ?? 0 > 0 ? { type: "auto", disable_parallel_tool_use: false } : undefined, messages: anthropic_messages, max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, @@ -154,21 +105,9 @@ export async function endpointAnthropic( stream.emitted("end"), ]); - // Stream end if (result === undefined) { - stream.receivedMessages.forEach((message) => { - console.log("--->" + message.id + "..." + message.stop_reason + ".." + message.type); - message.content.forEach((contentBlock) => { - if (contentBlock.type === "tool_use") { - console.log( - "Tool call:", - contentBlock.id + contentBlock.name + JSON.stringify(contentBlock.input) - ); - } - }); - }); - if ("tool_use" === stream.receivedMessages[0].stop_reason) { + // this should really create a new "Assistant" message with the tool id in it. const toolCalls: ToolCall[] = stream.receivedMessages[0].content .filter( (block): block is Anthropic.Messages.ContentBlock & { type: "tool_use" } => @@ -220,31 +159,41 @@ function addToolResults(messages: MessageParam[], toolResults: ToolResult[]): Me const assistantMessages: MessageParam[] = []; const userMessages: MessageParam[] = []; + const [toolUseBlocks, toolResultBlocks] = toolResults.reduce< + [Anthropic.Messages.ToolUseBlockParam[], Anthropic.Messages.ToolResultBlockParam[]] + >( + (acc, toolResult, index) => { + acc[0].push({ + type: "tool_use", + id: "tool_" + index, + name: toolResult.call.name, + input: toolResult.call.parameters, + }); + acc[1].push({ + type: "tool_result", + tool_use_id: "tool_" + index, + is_error: toolResult.status === "error", + content: + toolResult.status === "error" + ? JSON.stringify(toolResult.message) + : JSON.stringify("outputs" in toolResult ? toolResult.outputs : ""), + }); + return acc; + }, + [[], []] + ); + + console.log(JSON.stringify(toolUseBlocks)); + console.log(JSON.stringify(toolResultBlocks)); + assistantMessages.push({ role: "assistant", - content: [ - { - type: "tool_use", - id: "any_id", - name: toolResults[0].call.name, - input: toolResults[0].call.parameters, // Changed from JSON.stringify to direct object - }, - ], + content: toolUseBlocks, }); userMessages.push({ role: "user", - content: [ - { - type: "tool_result", - tool_use_id: "any_id", - is_error: toolResults[0].status === "error", - content: - toolResults[0].status === "error" - ? JSON.stringify(toolResults[0].message) // Include error message if it's an error - : JSON.stringify("outputs" in toolResults[0] ? toolResults[0].outputs : ""), // Otherwise include the output - }, - ], + content: toolResultBlocks, }); return [...messages, ...assistantMessages, ...userMessages]; diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index 6f3d13def1a..271f4899739 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -9,13 +9,15 @@ type GenerateContext = Omit & { messages: End export async function* generate( { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: GenerateContext, toolResults: ToolResult[], - preprompt?: string + preprompt?: string, + tools?: Tools[] ): AsyncIterable { for await (const output of await endpoint({ messages, preprompt, continueMessage: isContinue, generateSettings: assistant?.generateSettings, + tools, toolResults, isMultimodal: model.multimodal, conversationId: conv._id, diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index b5c9bc3ecf9..2bc2f42e99c 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -81,6 +81,11 @@ async function* textGenerationWithoutTitle( } const processedMessages = await preprocessMessages(messages, webSearchResult, convId); - yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt); + yield* generate( + { ...ctx, messages: processedMessages }, + toolResults, + preprompt, + model.tools ? await getTools(toolsPreference, ctx.assistant) : undefined + ); done.abort(); } From 506ecffa38dbec6d8d4fb6729bb587e63ba16574 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:00:54 +0000 Subject: [PATCH 08/14] remove merge error --- src/lib/server/endpoints/anthropic/endpointAnthropic.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 9a00a4a6440..184ce0336b3 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -16,7 +16,6 @@ import type { ToolResult, } from "$lib/types/Tool"; import type Anthropic from "@anthropic-ai/sdk"; -import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), From b233de77c219f4b34ccced9308d5a0af83b0c321 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:30:34 +0000 Subject: [PATCH 09/14] tidy up, isolate beta classes to utils --- .../endpoints/anthropic/endpointAnthropic.ts | 73 +++---------------- src/lib/server/endpoints/anthropic/utils.ts | 34 +++++++++ 2 files changed, 43 insertions(+), 64 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 184ce0336b3..52a90face43 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -3,9 +3,8 @@ import type { Endpoint } from "../endpoints"; import { env } from "$env/dynamic/private"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { createImageProcessorOptionsValidator } from "../images"; -import { endpointMessagesToAnthropicMessages } from "./utils"; +import { endpointMessagesToAnthropicMessages, addToolResults } from "./utils"; import { createDocumentProcessorOptionsValidator } from "../document"; -import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; import type { Tool, ToolCall, @@ -13,9 +12,9 @@ import type { ToolInputFile, ToolInputFixed, ToolInputOptional, - ToolResult, } from "$lib/types/Tool"; import type Anthropic from "@anthropic-ai/sdk"; +import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -67,7 +66,7 @@ export async function endpointAnthropic( // check if model has tools enabled // process toolresults and add them to the request // add a toolcall yield if the model stops for tool calling reasons. - return async ({ messages, preprompt, generateSettings, tools, toolResults }) => { + return async ({ messages, preprompt, generateSettings, tools = [], toolResults = [] }) => { let system = preprompt; if (messages?.[0]?.from === "system") { system = messages[0].content; @@ -77,26 +76,16 @@ export async function endpointAnthropic( const parameters = { ...model.parameters, ...generateSettings }; - let anthropicTools: Anthropic.Messages.Tool[] = []; - - if (model.tools && tools && tools.length > 0) { - anthropicTools = createAnthropicTools(tools); - } - console.log(JSON.stringify(messages)); - let anthropic_messages = await endpointMessagesToAnthropicMessages(messages, multimodal); - if (toolResults && toolResults.length > 0) { - anthropic_messages = addToolResults(anthropic_messages, toolResults); - } - - // console.log(JSON.stringify(anthropic_messages)); return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - - tools: anthropicTools, + tools: createAnthropicTools(tools), tool_choice: - tools?.length ?? 0 > 0 ? { type: "auto", disable_parallel_tool_use: false } : undefined, - messages: anthropic_messages, + tools.length > 0 ? { type: "auto", disable_parallel_tool_use: false } : undefined, + messages: addToolResults( + await endpointMessagesToAnthropicMessages(messages, multimodal), + toolResults + ) as MessageParam[], max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, top_p: parameters?.top_p, @@ -161,50 +150,6 @@ export async function endpointAnthropic( }; } -function addToolResults(messages: MessageParam[], toolResults: ToolResult[]): MessageParam[] { - const assistantMessages: MessageParam[] = []; - const userMessages: MessageParam[] = []; - - const [toolUseBlocks, toolResultBlocks] = toolResults.reduce< - [Anthropic.Messages.ToolUseBlockParam[], Anthropic.Messages.ToolResultBlockParam[]] - >( - (acc, toolResult, index) => { - acc[0].push({ - type: "tool_use", - id: "tool_" + index, - name: toolResult.call.name, - input: toolResult.call.parameters, - }); - acc[1].push({ - type: "tool_result", - tool_use_id: "tool_" + index, - is_error: toolResult.status === "error", - content: - toolResult.status === "error" - ? JSON.stringify(toolResult.message) - : JSON.stringify("outputs" in toolResult ? toolResult.outputs : ""), - }); - return acc; - }, - [[], []] - ); - - console.log(JSON.stringify(toolUseBlocks)); - console.log(JSON.stringify(toolResultBlocks)); - - assistantMessages.push({ - role: "assistant", - content: toolUseBlocks, - }); - - userMessages.push({ - role: "user", - content: toolResultBlocks, - }); - - return [...messages, ...assistantMessages, ...userMessages]; -} - function createAnthropicTools(tools: Tool[]): Anthropic.Messages.Tool[] { return tools.map((tool) => { const properties = tool.inputs.reduce((acc, input) => { diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index 0239e426300..f8a42eba14a 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -7,6 +7,7 @@ import type { BetaMessageParam, BetaBase64PDFBlock, } from "@anthropic-ai/sdk/resources/beta/messages/messages.mjs"; +import type { ToolResult } from "$lib/types/Tool"; export async function fileToImageBlock( file: MessageFile, @@ -74,3 +75,36 @@ export async function endpointMessagesToAnthropicMessages( }) ); } + +export function addToolResults( + messages: BetaMessageParam[], + toolResults: ToolResult[] +): BetaMessageParam[] { + const id = crypto.randomUUID(); + if (toolResults.length === 0) { + return messages; + } + return [ + ...messages, + { + role: "assistant", + content: toolResults.map((result, index) => ({ + type: "tool_use", + id: `tool_${index}_${id}`, + name: result.call.name, + input: result.call.parameters, + })), + }, + { + role: "user", + content: toolResults.map((result, index) => ({ + type: "tool_result", + tool_use_id: `tool_${index}_${id}`, + is_error: result.status === "error", + content: JSON.stringify( + result.status === "error" ? result.message : "outputs" in result ? result.outputs : "" + ), + })), + }, + ]; +} From ee6107a411d90d6e45fb51b36ccfb66129acf9cc Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 19:02:24 +0000 Subject: [PATCH 10/14] anthropic tool calling support. --- .../endpoints/anthropic/endpointAnthropic.ts | 33 ++++++------------ src/lib/server/endpoints/anthropic/utils.ts | 34 ++++++++++++------- src/lib/server/textGeneration/generate.ts | 4 +-- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 52a90face43..3a59b82fd9e 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -61,12 +61,14 @@ export async function endpointAnthropic( defaultQuery, }); - // get tool results and tools from this call - // convert from library to anthropic format of tools - // check if model has tools enabled - // process toolresults and add them to the request - // add a toolcall yield if the model stops for tool calling reasons. - return async ({ messages, preprompt, generateSettings, tools = [], toolResults = [] }) => { + return async ({ + messages, + preprompt, + generateSettings, + conversationId, + tools = [], + toolResults = [], + }) => { let system = preprompt; if (messages?.[0]?.from === "system") { system = messages[0].content; @@ -83,7 +85,7 @@ export async function endpointAnthropic( tool_choice: tools.length > 0 ? { type: "auto", disable_parallel_tool_use: false } : undefined, messages: addToolResults( - await endpointMessagesToAnthropicMessages(messages, multimodal), + await endpointMessagesToAnthropicMessages(messages, multimodal, conversationId), toolResults ) as MessageParam[], max_tokens: parameters?.max_new_tokens, @@ -94,11 +96,7 @@ export async function endpointAnthropic( system, }); while (true) { - const result = await Promise.race([ - // stream.emitted("inputJson"), - stream.emitted("text"), - stream.emitted("end"), - ]); + const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]); if (result === undefined) { if ("tool_use" === stream.receivedMessages[0].stop_reason) { @@ -134,17 +132,6 @@ export async function endpointAnthropic( return; } - - yield { - token: { - id: tokenId++, - text: result as unknown as string, - special: false, - logprob: 0, - }, - generated_text: null, - details: null, - } satisfies TextGenerationStreamOutput; } })(); }; diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts index f8a42eba14a..c935ea43d95 100644 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -8,12 +8,15 @@ import type { BetaBase64PDFBlock, } from "@anthropic-ai/sdk/resources/beta/messages/messages.mjs"; import type { ToolResult } from "$lib/types/Tool"; +import { downloadFile } from "$lib/server/files/downloadFile"; +import type { ObjectId } from "mongodb"; export async function fileToImageBlock( file: MessageFile, opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> ): Promise { const processor = makeImageProcessor(opts); + const { image, mime } = await processor(file); return { @@ -49,7 +52,8 @@ export async function endpointMessagesToAnthropicMessages( multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">; document?: FileProcessorOptions<"application/pdf">; - } + }, + conversationId?: ObjectId | undefined ): Promise { return await Promise.all( messages @@ -58,17 +62,23 @@ export async function endpointMessagesToAnthropicMessages( return { role: message.from, content: [ - ...(await Promise.all( - (message.files ?? []).map(async (file) => { - if (file.mime.startsWith("image/")) { - return fileToImageBlock(file, multimodal.image); - } else if (file.mime === "application/pdf" && multimodal.document) { - return fileToDocumentBlock(file, multimodal.document); - } else { - throw new Error(`Unsupported file type: ${file.mime}`); - } - }) - )), + ...(message.from === "user" + ? await Promise.all( + (message.files ?? []).map(async (file) => { + if (file.type === "hash" && conversationId) { + file = await downloadFile(file.value, conversationId); + } + + if (file.mime.startsWith("image/")) { + return fileToImageBlock(file, multimodal.image); + } else if (file.mime === "application/pdf" && multimodal.document) { + return fileToDocumentBlock(file, multimodal.document); + } else { + throw new Error(`Unsupported file type: ${file.mime}`); + } + }) + ) + : []), { type: "text", text: message.content }, ], }; diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index 271f4899739..0dda6d34fa7 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -1,4 +1,4 @@ -import type { ToolResult } from "$lib/types/Tool"; +import type { ToolResult, Tool } from "$lib/types/Tool"; import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate"; import { AbortedGenerations } from "../abortedGenerations"; import type { TextGenerationContext } from "./types"; @@ -10,7 +10,7 @@ export async function* generate( { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: GenerateContext, toolResults: ToolResult[], preprompt?: string, - tools?: Tools[] + tools?: Tool[] ): AsyncIterable { for await (const output of await endpoint({ messages, From a8bac5897d500afccceb84af9ef04d97fdf43ed2 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 21:08:27 +0000 Subject: [PATCH 11/14] improve handling of directlyAnswer tool --- .../endpoints/anthropic/endpointAnthropic.ts | 43 ++++++++++--------- src/lib/server/textGeneration/index.ts | 14 +++--- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 3a59b82fd9e..5d475d14711 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -15,6 +15,7 @@ import type { } from "$lib/types/Tool"; import type Anthropic from "@anthropic-ai/sdk"; import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs"; +import directlyAnswer from "$lib/server/tools/directlyAnswer"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -138,26 +139,28 @@ export async function endpointAnthropic( } function createAnthropicTools(tools: Tool[]): Anthropic.Messages.Tool[] { - return tools.map((tool) => { - const properties = tool.inputs.reduce((acc, input) => { - acc[input.name] = convertToolInputToJSONSchema(input); - return acc; - }, {} as Record); - - const required = tool.inputs - .filter((input) => input.paramType === "required") - .map((input) => input.name); - - return { - name: tool.name, - description: tool.description, - input_schema: { - type: "object", - properties, - required: required.length > 0 ? required : undefined, - }, - }; - }); + return tools + .filter((tool) => tool.name !== directlyAnswer.name) + .map((tool) => { + const properties = tool.inputs.reduce((acc, input) => { + acc[input.name] = convertToolInputToJSONSchema(input); + return acc; + }, {} as Record); + + const required = tool.inputs + .filter((input) => input.paramType === "required") + .map((input) => input.name); + + return { + name: tool.name, + description: tool.description, + input_schema: { + type: "object", + properties, + required: required.length > 0 ? required : undefined, + }, + }; + }); } function convertToolInputToJSONSchema(input: ToolInput): Record { diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index 2bc2f42e99c..48eb8f32532 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -20,6 +20,7 @@ import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators"; import type { TextGenerationContext } from "./types"; import type { ToolResult } from "$lib/types/Tool"; import { toolHasName } from "../tools/utils"; +import directlyAnswer from "../tools/directlyAnswer"; async function* keepAlive(done: AbortSignal): AsyncGenerator { while (!done.aborted) { @@ -73,19 +74,16 @@ async function* textGenerationWithoutTitle( } let toolResults: ToolResult[] = []; + let tools = undefined; if (model.tools) { - const tools = await getTools(toolsPreference, ctx.assistant); - const toolCallsRequired = tools.some((tool) => !toolHasName("directly_answer", tool)); + tools = await getTools(toolsPreference, ctx.assistant); + const toolCallsRequired = tools.some((tool) => !toolHasName(directlyAnswer.name, tool)); if (toolCallsRequired) toolResults = yield* runTools(ctx, tools, preprompt); + else tools = undefined; } const processedMessages = await preprocessMessages(messages, webSearchResult, convId); - yield* generate( - { ...ctx, messages: processedMessages }, - toolResults, - preprompt, - model.tools ? await getTools(toolsPreference, ctx.assistant) : undefined - ); + yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt, tools); done.abort(); } From 87b57f3dae33d51c611696628e7b0b43c73b9a27 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Tue, 26 Nov 2024 19:16:23 +0000 Subject: [PATCH 12/14] fix streaming --- .../server/endpoints/anthropic/endpointAnthropic.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 5d475d14711..45f08320891 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -133,6 +133,17 @@ export async function endpointAnthropic( return; } + // Text delta + yield { + token: { + id: tokenId++, + text: result as unknown as string, + special: false, + logprob: 0, + }, + generated_text: null, + details: null, + } satisfies TextGenerationStreamOutput; } })(); }; From 0c9abdf6009f2be6e136f346ba3b1dd31d07c2f2 Mon Sep 17 00:00:00 2001 From: evalstate <1936278+evalstate@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:14:08 +0000 Subject: [PATCH 13/14] slight tidy up to tools flow handling --- src/lib/server/textGeneration/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index 48eb8f32532..cffffe26c8a 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -74,13 +74,13 @@ async function* textGenerationWithoutTitle( } let toolResults: ToolResult[] = []; - let tools = undefined; + let tools = model.tools ? await getTools(toolsPreference, ctx.assistant) : undefined; - if (model.tools) { - tools = await getTools(toolsPreference, ctx.assistant); + if (tools) { const toolCallsRequired = tools.some((tool) => !toolHasName(directlyAnswer.name, tool)); - if (toolCallsRequired) toolResults = yield* runTools(ctx, tools, preprompt); - else tools = undefined; + if (toolCallsRequired) { + toolResults = yield* runTools(ctx, tools, preprompt); + } else tools = undefined; } const processedMessages = await preprocessMessages(messages, webSearchResult, convId); From e0adc6890baed75d85281858a0285fa3396291bb Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 3 Jan 2025 15:36:59 +0000 Subject: [PATCH 14/14] fix: dont pass tools in final generation, instead deduce tools from tool results --- src/lib/server/endpoints/anthropic/endpointAnthropic.ts | 8 ++++++++ src/lib/server/textGeneration/index.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 45f08320891..fc085baecd0 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -76,6 +76,14 @@ export async function endpointAnthropic( } let tokenId = 0; + if (tools.length === 0 && toolResults.length > 0) { + const toolNames = new Set(toolResults.map((tool) => tool.call.name)); + tools = Array.from(toolNames).map((name) => ({ + name, + description: "", + inputs: [], + })) as unknown as Tool[]; + } const parameters = { ...model.parameters, ...generateSettings }; diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index 52030aff1df..0142acfbb52 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -84,6 +84,6 @@ async function* textGenerationWithoutTitle( } const processedMessages = await preprocessMessages(messages, webSearchResult, convId); - yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt, tools); + yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt); done.abort(); }