From 5d265bf73b36a67d580b4243b75cb5664377541e Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Fri, 10 Jan 2025 17:26:43 -0800 Subject: [PATCH 1/3] draft --- js/src/client.ts | 4 +++- js/src/evaluation/_runner.ts | 2 +- js/src/schemas.ts | 2 ++ js/src/tests/evaluate_attachments.int.test.ts | 8 ++++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/js/src/client.ts b/js/src/client.ts index d9aa11ca4..23553e20f 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -60,7 +60,7 @@ import { RunEvaluator, } from "./evaluation/evaluator.js"; import { __version__ } from "./index.js"; -import { assertUuid } from "./utils/_uuid.js"; +import { assertUuid } from "./utils/uuid.js"; import { warnOnce } from "./utils/warn.js"; import { parsePromptIdentifier } from "./utils/prompts.js"; import { raiseForStatus } from "./utils/error.js"; @@ -2770,6 +2770,7 @@ export class Client implements LangSmithTracingClientInterface { (acc, [key, value]) => { acc[key.slice("attachment.".length)] = { presigned_url: value.presigned_url, + mime_type: value.mime_type || undefined, }; return acc; }, @@ -2867,6 +2868,7 @@ export class Client implements LangSmithTracingClientInterface { (acc, [key, value]) => { acc[key.slice("attachment.".length)] = { presigned_url: value.presigned_url, + mime_type: value.mime_type || undefined, }; return acc; }, diff --git a/js/src/evaluation/_runner.ts b/js/src/evaluation/_runner.ts index 6c74afa34..e9bed2e4a 100644 --- a/js/src/evaluation/_runner.ts +++ b/js/src/evaluation/_runner.ts @@ -9,7 +9,7 @@ import { } from "../schemas.js"; import { traceable } from "../traceable.js"; import { getDefaultRevisionId, getGitInfo } from "../utils/_git.js"; -import { assertUuid } from "../utils/_uuid.js"; +import { assertUuid } from "../utils/uuid.js"; import { AsyncCaller } from "../utils/async_caller.js"; import { atee } from "../utils/atee.js"; import { getLangChainEnvVarsMetadata } from "../utils/env.js"; diff --git a/js/src/schemas.ts b/js/src/schemas.ts index 9fe4e8e16..d822e8392 100644 --- a/js/src/schemas.ts +++ b/js/src/schemas.ts @@ -65,6 +65,7 @@ export interface BaseExample { export interface AttachmentInfo { presigned_url: string; + mime_type?: string; } export type AttachmentData = Uint8Array | ArrayBuffer; @@ -300,6 +301,7 @@ export interface Example extends BaseExample { interface RawAttachmentInfo { presigned_url: string; s3_url: string; + mime_type?: string; } export interface RawExample extends BaseExample { id: string; diff --git a/js/src/tests/evaluate_attachments.int.test.ts b/js/src/tests/evaluate_attachments.int.test.ts index 0a64a54f9..961a6c263 100644 --- a/js/src/tests/evaluate_attachments.int.test.ts +++ b/js/src/tests/evaluate_attachments.int.test.ts @@ -40,6 +40,10 @@ test("evaluate can handle examples with attachments", async () => { const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); + const attachmentMimeType = config?.attachments?.["image"].mime_type; + if (attachmentMimeType !== "image/png") { + throw new Error("Image attachment has incorrect mime type"); + } const attachmentData: Uint8Array | undefined = config?.attachments?.[ "image" ].presigned_url @@ -61,6 +65,10 @@ test("evaluate can handle examples with attachments", async () => { const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); + const attachmentMimeType = attachments?.["image"].mime_type; + if (attachmentMimeType !== "image/png") { + throw new Error("Image attachment has incorrect mime type"); + } const attachmentData: Uint8Array | undefined = attachments?.["image"] .presigned_url ? new Uint8Array( From 65a4e743657885dafd4e4ceff14fe559216f4497 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 13 Jan 2025 08:38:09 -0800 Subject: [PATCH 2/3] comments --- js/src/client.ts | 2 +- js/src/tests/evaluate_attachments.int.test.ts | 52 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/js/src/client.ts b/js/src/client.ts index 23553e20f..90aa1bccc 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -2770,7 +2770,7 @@ export class Client implements LangSmithTracingClientInterface { (acc, [key, value]) => { acc[key.slice("attachment.".length)] = { presigned_url: value.presigned_url, - mime_type: value.mime_type || undefined, + mime_type: value.mime_type, }; return acc; }, diff --git a/js/src/tests/evaluate_attachments.int.test.ts b/js/src/tests/evaluate_attachments.int.test.ts index 961a6c263..8a359e0f5 100644 --- a/js/src/tests/evaluate_attachments.int.test.ts +++ b/js/src/tests/evaluate_attachments.int.test.ts @@ -34,13 +34,13 @@ test("evaluate can handle examples with attachments", async () => { config?: TargetConfigT ) => { // Verify we receive the attachment data - if (!config?.attachments?.["image"]) { + if (!config?.attachments?.image) { throw new Error("Image attachment not found"); } const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentMimeType = config?.attachments?.["image"].mime_type; + const attachmentMimeType = config?.attachments?.image.mime_type; if (attachmentMimeType !== "image/png") { throw new Error("Image attachment has incorrect mime type"); } @@ -48,8 +48,8 @@ test("evaluate can handle examples with attachments", async () => { "image" ].presigned_url ? new Uint8Array( - (await fetch(config?.attachments?.["image"].presigned_url).then( - (res) => res.arrayBuffer() + (await fetch(config?.attachments?.image.presigned_url).then((res) => + res.arrayBuffer() )) as ArrayBuffer ) : undefined; @@ -61,18 +61,18 @@ test("evaluate can handle examples with attachments", async () => { const customEvaluator = async ({ attachments }: { attachments?: any }) => { expect(attachments).toBeDefined(); - expect(attachments?.["image"]).toBeDefined(); + expect(attachments?.image).toBeDefined(); const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentMimeType = attachments?.["image"].mime_type; + const attachmentMimeType = attachments?.image.mime_type; if (attachmentMimeType !== "image/png") { throw new Error("Image attachment has incorrect mime type"); } - const attachmentData: Uint8Array | undefined = attachments?.["image"] + const attachmentData: Uint8Array | undefined = attachments?.image .presigned_url ? new Uint8Array( - (await fetch(attachments?.["image"].presigned_url).then((res) => + (await fetch(attachments?.image.presigned_url).then((res) => res.arrayBuffer() )) as ArrayBuffer ) @@ -142,14 +142,14 @@ test("evaluate with attachments not in target function", async () => { const customEvaluator = async ({ attachments }: { attachments?: any }) => { expect(attachments).toBeDefined(); - expect(attachments?.["image"]).toBeDefined(); + expect(attachments?.image).toBeDefined(); const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentData: Uint8Array | undefined = attachments?.["image"] + const attachmentData: Uint8Array | undefined = attachments?.image .presigned_url ? new Uint8Array( - (await fetch(attachments?.["image"].presigned_url).then((res) => + (await fetch(attachments?.image.presigned_url).then((res) => res.arrayBuffer() )) as ArrayBuffer ) @@ -218,7 +218,7 @@ test("multiple evaluators with attachments", async () => { config?: TargetConfigT ) => { // Verify we receive the attachment data - if (!config?.attachments?.["image"]) { + if (!config?.attachments?.image) { throw new Error("Image attachment not found"); } const expectedData = new Uint8Array( @@ -228,8 +228,8 @@ test("multiple evaluators with attachments", async () => { "image" ].presigned_url ? new Uint8Array( - (await fetch(config?.attachments?.["image"].presigned_url).then( - (res) => res.arrayBuffer() + (await fetch(config?.attachments?.image.presigned_url).then((res) => + res.arrayBuffer() )) as ArrayBuffer ) : undefined; @@ -241,14 +241,14 @@ test("multiple evaluators with attachments", async () => { const customEvaluatorOne = async ({ attachments }: { attachments?: any }) => { expect(attachments).toBeDefined(); - expect(attachments?.["image"]).toBeDefined(); + expect(attachments?.image).toBeDefined(); const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentData: Uint8Array | undefined = attachments?.["image"] + const attachmentData: Uint8Array | undefined = attachments?.image .presigned_url ? new Uint8Array( - (await fetch(attachments?.["image"].presigned_url).then((res) => + (await fetch(attachments?.image.presigned_url).then((res) => res.arrayBuffer() )) as ArrayBuffer ) @@ -264,14 +264,14 @@ test("multiple evaluators with attachments", async () => { const customEvaluatorTwo = async ({ attachments }: { attachments?: any }) => { expect(attachments).toBeDefined(); - expect(attachments?.["image"]).toBeDefined(); + expect(attachments?.image).toBeDefined(); const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentData: Uint8Array | undefined = attachments?.["image"] + const attachmentData: Uint8Array | undefined = attachments?.image .presigned_url ? new Uint8Array( - (await fetch(attachments?.["image"].presigned_url).then((res) => + (await fetch(attachments?.image.presigned_url).then((res) => res.arrayBuffer() )) as ArrayBuffer ) @@ -337,7 +337,7 @@ test("evaluate with attachments runnable target function", async () => { await client.uploadExamplesMultipart(dataset.id, [example]); const myFunction = async (_input: any, config?: any) => { - if (!config?.attachments?.["image"]) { + if (!config?.attachments?.image) { throw new Error("Image attachment not found"); } const expectedData = new Uint8Array( @@ -347,8 +347,8 @@ test("evaluate with attachments runnable target function", async () => { "image" ].presigned_url ? new Uint8Array( - (await fetch(config?.attachments?.["image"].presigned_url).then( - (res) => res.arrayBuffer() + (await fetch(config?.attachments?.image.presigned_url).then((res) => + res.arrayBuffer() )) as ArrayBuffer ) : undefined; @@ -363,14 +363,14 @@ test("evaluate with attachments runnable target function", async () => { const customEvaluator = async ({ attachments }: { attachments?: any }) => { expect(attachments).toBeDefined(); - expect(attachments?.["image"]).toBeDefined(); + expect(attachments?.image).toBeDefined(); const expectedData = new Uint8Array( Buffer.from("fake image data for testing") ); - const attachmentData: Uint8Array | undefined = attachments?.["image"] + const attachmentData: Uint8Array | undefined = attachments?.image .presigned_url ? new Uint8Array( - (await fetch(attachments?.["image"].presigned_url).then((res) => + (await fetch(attachments?.image.presigned_url).then((res) => res.arrayBuffer() )) as ArrayBuffer ) From cade9bace3455e8280be4af81fc546477889af6d Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 13 Jan 2025 08:44:26 -0800 Subject: [PATCH 3/3] fmt --- js/src/client.ts | 2 +- js/src/evaluation/_runner.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/client.ts b/js/src/client.ts index 90aa1bccc..31704d448 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -60,7 +60,7 @@ import { RunEvaluator, } from "./evaluation/evaluator.js"; import { __version__ } from "./index.js"; -import { assertUuid } from "./utils/uuid.js"; +import { assertUuid } from "./utils/_uuid.js"; import { warnOnce } from "./utils/warn.js"; import { parsePromptIdentifier } from "./utils/prompts.js"; import { raiseForStatus } from "./utils/error.js"; diff --git a/js/src/evaluation/_runner.ts b/js/src/evaluation/_runner.ts index e9bed2e4a..6c74afa34 100644 --- a/js/src/evaluation/_runner.ts +++ b/js/src/evaluation/_runner.ts @@ -9,7 +9,7 @@ import { } from "../schemas.js"; import { traceable } from "../traceable.js"; import { getDefaultRevisionId, getGitInfo } from "../utils/_git.js"; -import { assertUuid } from "../utils/uuid.js"; +import { assertUuid } from "../utils/_uuid.js"; import { AsyncCaller } from "../utils/async_caller.js"; import { atee } from "../utils/atee.js"; import { getLangChainEnvVarsMetadata } from "../utils/env.js";