From e86dc3c983683450ace432b762a2fd83caf95d2e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:13:08 +0200 Subject: [PATCH 01/47] feat: add fetchScores method to LangfuseCoreStateless class --- langfuse-core/src/index.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index e0d5089b..8fef37a5 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -45,6 +45,8 @@ import { type ChatMessage, type GetLangfuseSessionsQuery, type GetLangfuseSessionsResponse, + type GetLangfuseScoresQuery, + type GetLangfuseScoresResponse, } from "./types"; import { generateUUID, @@ -338,6 +340,16 @@ abstract class LangfuseCoreStateless { return { data, meta }; } + async fetchScores(query?: GetLangfuseScoresQuery): Promise { + const res = await this.fetch( + `${this.baseUrl}/api/public/scores?${encodeQueryParams(query)}`, + this._getFetchOptions({ method: "GET" }) + ); + // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes + const { data, meta } = (await res.json()) as GetLangfuseScoresResponse; + return { data, meta }; + } + async getDatasetRun(params: GetLangfuseDatasetRunParams): Promise { const encodedDatasetName = encodeURIComponent(params.datasetName); const encodedRunName = encodeURIComponent(params.runName); From 376712ca734d6eb2deb59df70fb73b9813f49774 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:13:13 +0200 Subject: [PATCH 02/47] chore: Add types for fetching Langfuse scores API endpoint --- langfuse-core/src/types.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/langfuse-core/src/types.ts b/langfuse-core/src/types.ts index 3fa206dd..016f305b 100644 --- a/langfuse-core/src/types.ts +++ b/langfuse-core/src/types.ts @@ -99,6 +99,10 @@ export type GetLangfuseSessionsQuery = FixTypes; +export type GetLangfuseScoresQuery = FixTypes; +export type GetLangfuseScoresResponse = FixTypes< + paths["/api/public/scores"]["get"]["responses"]["200"]["content"]["application/json"] +>; export type GetLangfuseDatasetParams = FixTypes< paths["/api/public/v2/datasets/{datasetName}"]["get"]["parameters"]["path"] >; From 628a175df784f0623427388906100d98d16237aa Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:15:59 +0200 Subject: [PATCH 03/47] feat: add test for creating and fetching scores in Langfuse Node.js integration test --- .../langfuse-integration-node.spec.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 20333ab9..910e70c7 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -551,6 +551,22 @@ describe("Langfuse Node.js", () => { }); }); + it("create and fetch scores", async () => { + const traceName = utils.generateUUID(); + const trace = langfuse.trace({ + name: traceName, + }); + const score = trace.score({ + name: "quality", + value: 1, + comment: "Factually correct", + }); + await langfuse.flushAsync(); + + const scores = await langfuse.fetchScores({ name: "quality" }); + expect(scores.data).toContainEqual(expect.objectContaining({ id: score.id, name: "quality" })); + }); + it("create 3 traces with different timestamps and fetch the middle one using to and from timestamp", async () => { const traceName = utils.generateUUID(); const traceParams = [ From a6f8893fcc6f1acd716cd71eef6bb3b61f55b2cc Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:45:52 +0200 Subject: [PATCH 04/47] fix: update expect assertion in Langfuse Node.js integration test --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 910e70c7..98cb1938 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -564,7 +564,7 @@ describe("Langfuse Node.js", () => { await langfuse.flushAsync(); const scores = await langfuse.fetchScores({ name: "quality" }); - expect(scores.data).toContainEqual(expect.objectContaining({ id: score.id, name: "quality" })); + expect(scores.data).toContainEqual(expect.objectContaining({ traceId: score.id, name: "quality" })); }); it("create 3 traces with different timestamps and fetch the middle one using to and from timestamp", async () => { From f35c15afcf956ced82df1d0b436d39dc0f1cf0b3 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:43:23 +0200 Subject: [PATCH 05/47] feat: add test for creating and fetching prompts in Langfuse Node.js integration test --- integration-test/langfuse-integration-node.spec.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 98cb1938..0e76805b 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -643,4 +643,15 @@ describe("Langfuse Node.js", () => { const sessions = await langfuse.fetchSessions(); expect(sessions.data).toContainEqual(expect.objectContaining({ id: sessionId })); }); + + it("create and fetch prompts", async () => { + const promptName = utils.generateUUID(); + await langfuse.createPrompt({ + name: promptName, + prompt: "This is a prompt", + }); + + const prompts = await langfuse.fetchPrompts({ name: promptName }); + expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName })); + }); }); From 9525a21edc21285fdf4e8e587928977b11d7df32 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:43:27 +0200 Subject: [PATCH 06/47] feat: add fetchPrompts method to LangfuseCoreStateless class --- langfuse-core/src/index.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 8fef37a5..0d8f7b00 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -47,6 +47,8 @@ import { type GetLangfuseSessionsResponse, type GetLangfuseScoresQuery, type GetLangfuseScoresResponse, + type GetLangfusePromptsQuery, + type GetLangfusePromptsResponse, } from "./types"; import { generateUUID, @@ -350,6 +352,16 @@ abstract class LangfuseCoreStateless { return { data, meta }; } + async fetchPrompts(query?: GetLangfusePromptsQuery): Promise { + const res = await this.fetch( + `${this.baseUrl}/api/public/prompts?${encodeQueryParams(query)}`, + this._getFetchOptions({ method: "GET" }) + ); + // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes + const { data, meta } = (await res.json()) as GetLangfusePromptsResponse; + return { data, meta }; + } + async getDatasetRun(params: GetLangfuseDatasetRunParams): Promise { const encodedDatasetName = encodeURIComponent(params.datasetName); const encodedRunName = encodeURIComponent(params.runName); From 9e28453a7b2dac9f3cdd4c21b258b69ec1faa050 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:43:31 +0200 Subject: [PATCH 07/47] chore: Add types for fetching Langfuse prompts API endpoint --- langfuse-core/src/types.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/langfuse-core/src/types.ts b/langfuse-core/src/types.ts index 016f305b..e9c2882c 100644 --- a/langfuse-core/src/types.ts +++ b/langfuse-core/src/types.ts @@ -103,6 +103,10 @@ export type GetLangfuseScoresQuery = FixTypes; +export type GetLangfusePromptsQuery = FixTypes; +export type GetLangfusePromptsResponse = FixTypes< + paths["/api/public/v2/prompts"]["get"]["responses"]["200"]["content"]["application/json"] +>; export type GetLangfuseDatasetParams = FixTypes< paths["/api/public/v2/datasets/{datasetName}"]["get"]["parameters"]["path"] >; From b6ae6fa1a42063779e1619f4cc0b081735c3fe1d Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:06:31 +0200 Subject: [PATCH 08/47] chore: Update Langfuse integration test to fetch prompts without specifying a name --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 0e76805b..25a5cb47 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -651,7 +651,7 @@ describe("Langfuse Node.js", () => { prompt: "This is a prompt", }); - const prompts = await langfuse.fetchPrompts({ name: promptName }); + const prompts = await langfuse.fetchPrompts(); expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName })); }); }); From edf042b675e518cf84bf1364fae1f9fc56f6928d Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:14:05 +0200 Subject: [PATCH 09/47] refactor: update LangfuseCoreStateless fetchPrompts method to use v2 API endpoint --- langfuse-core/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 0d8f7b00..60eb892f 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -354,7 +354,7 @@ abstract class LangfuseCoreStateless { async fetchPrompts(query?: GetLangfusePromptsQuery): Promise { const res = await this.fetch( - `${this.baseUrl}/api/public/prompts?${encodeQueryParams(query)}`, + `${this.baseUrl}/api/public/v2/prompts?${encodeQueryParams(query)}`, this._getFetchOptions({ method: "GET" }) ); // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes From 1b052039d6f8944c907c9a6fe30dbde77647371a Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:48:41 +0200 Subject: [PATCH 10/47] feat: add multiple prompts creation and fetching in Langfuse Node.js integration test --- .../langfuse-integration-node.spec.ts | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 25a5cb47..1234e824 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -645,13 +645,46 @@ describe("Langfuse Node.js", () => { }); it("create and fetch prompts", async () => { - const promptName = utils.generateUUID(); + const promptName1 = utils.generateUUID(); + const promptName2 = utils.generateUUID(); + const promptName3 = utils.generateUUID(); + + // Create multiple prompts await langfuse.createPrompt({ - name: promptName, - prompt: "This is a prompt", + name: promptName1, + prompt: "This is prompt 1", + }); + await langfuse.createPrompt({ + name: promptName2, + prompt: "This is prompt 2", + }); + await langfuse.createPrompt({ + name: promptName3, + prompt: "This is prompt 3", + }); + + // Create multiple versions of the same prompt + await langfuse.createPrompt({ + name: promptName1, + prompt: "This is prompt 1 version 2", + }); + await langfuse.createPrompt({ + name: promptName1, + prompt: "This is prompt 1 version 3", }); const prompts = await langfuse.fetchPrompts(); - expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName })); + expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName1 })); + expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName2 })); + expect(prompts.data).toContainEqual(expect.objectContaining({ name: promptName3 })); + + // Fetch specific versions of the prompt + const prompt1Version1 = await langfuse.getPrompt(promptName1, 1); + const prompt1Version2 = await langfuse.getPrompt(promptName1, 2); + const prompt1Version3 = await langfuse.getPrompt(promptName1, 3); + + expect(prompt1Version1.prompt).toEqual("This is prompt 1"); + expect(prompt1Version2.prompt).toEqual("This is prompt 1 version 2"); + expect(prompt1Version3.prompt).toEqual("This is prompt 1 version 3"); }); }); From a8641a96776d04c6902b88dce633c8b8a1d50ff5 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:43:40 +0200 Subject: [PATCH 11/47] feat: add getDatasetItems method to LangfuseCoreStateless class --- langfuse-core/src/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 60eb892f..388cf462 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -428,6 +428,13 @@ abstract class LangfuseCoreStateless { ); } + async getDatasetItems(query: GetLangfuseDatasetItemsQuery): Promise { + return this.fetch( + `${this.baseUrl}/api/public/dataset-items?${encodeQueryParams(query)}`, + this._getFetchOptions({ method: "GET" }) + ).then((res) => res.json()); + } + protected _parsePayload(response: any): any { try { return JSON.parse(response); From a3c1a8416aa192e4908a666f8ad248cf4eace007 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:43:50 +0200 Subject: [PATCH 12/47] feat: add create and fetch dataset items in Langfuse Node.js integration test --- .../langfuse-integration-node.spec.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 1234e824..170d37d9 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -5,6 +5,7 @@ import Langfuse from "../langfuse-node"; import axios from "axios"; import { LANGFUSE_BASEURL, getHeaders } from "./integration-utils"; import { utils } from "../langfuse-core/src"; +import exp from "constants"; describe("Langfuse Node.js", () => { let langfuse: Langfuse; @@ -687,4 +688,54 @@ describe("Langfuse Node.js", () => { expect(prompt1Version2.prompt).toEqual("This is prompt 1 version 2"); expect(prompt1Version3.prompt).toEqual("This is prompt 1 version 3"); }); + + it("create and fetch dataset items", async () => { + const datasetName = utils.generateUUID(); + const datasetItemName = utils.generateUUID(); + langfuse.createDataset({ + name: datasetName, + description: "My first dataset", + metadata: { + author: "Alice", + date: "2022-01-01", + type: "benchmark", + }, + }); + langfuse.createDatasetItem({ + datasetName: datasetName, + input: { + text: "hello world", + }, + expectedOutput: { + text: "hello world", + }, + metadata: { + model: "llama3", + }, + }); + langfuse.createDatasetItem({ + datasetName: datasetName, + input: { + text: "goodbye world", + }, + expectedOutput: { + text: "goodbye world", + }, + metadata: { + model: "llama4", + }, + }); + await langfuse.flushAsync(); + + const datasetItems = await langfuse.getDatasetItems({ name: datasetItemName }); + + expect(datasetItems.data).toHaveLength(2); + expect(datasetItems.data[0]).toContainEqual(expect.objectContaining({ name: datasetItemName })); + expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); + expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); + expect(datasetItems.data[0].metadata).toEqual({ model: "llama3" }); + expect(datasetItems.data[1].input).toEqual({ text: "goodbye world" }); + expect(datasetItems.data[1].expectedOutput).toEqual({ text: "goodbye world" }); + expect(datasetItems.data[1].metadata).toEqual({ model: "llama4" }); + }); }); From 8adcc27bf5b72295108d881ae7fe7c00e8eabcfd Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:48:32 +0200 Subject: [PATCH 13/47] refactor: Update LangfuseCoreStateless fetchPrompts method to use v2 API endpoint --- integration-test/langfuse-integration-node.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 170d37d9..5aa672d5 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -691,7 +691,6 @@ describe("Langfuse Node.js", () => { it("create and fetch dataset items", async () => { const datasetName = utils.generateUUID(); - const datasetItemName = utils.generateUUID(); langfuse.createDataset({ name: datasetName, description: "My first dataset", @@ -727,10 +726,10 @@ describe("Langfuse Node.js", () => { }); await langfuse.flushAsync(); - const datasetItems = await langfuse.getDatasetItems({ name: datasetItemName }); + const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); expect(datasetItems.data).toHaveLength(2); - expect(datasetItems.data[0]).toContainEqual(expect.objectContaining({ name: datasetItemName })); + expect(datasetItems.data[0]).toContainEqual(expect.objectContaining({ datasetName: datasetName })); expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); expect(datasetItems.data[0].metadata).toEqual({ model: "llama3" }); From d47a5acecc5052f210c69d117ff74311a72d0d02 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:48:43 +0200 Subject: [PATCH 14/47] refactor: Update LangfuseCoreStateless fetchPrompts method to use v2 API endpoint --- langfuse-core/src/index.ts | 9 +++++++++ langfuse-core/src/types.ts | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 388cf462..f583c27b 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -49,6 +49,8 @@ import { type GetLangfuseScoresResponse, type GetLangfusePromptsQuery, type GetLangfusePromptsResponse, + type GetLangfuseDatasetsQuery, + type GetLangfuseDatasetsResponse, } from "./types"; import { generateUUID, @@ -896,6 +898,13 @@ export abstract class LangfuseCore extends LangfuseCoreStateless { return this; } + async getDatasets(body: GetLangfuseDatasetsQuery): Promise { + return this.fetch( + `${this.baseUrl}/api/public/datasets?${encodeQueryParams(body)}`, + this._getFetchOptions({ method: "GET" }) + ).then((res) => res.json()); + } + async getDataset( name: string, options?: { diff --git a/langfuse-core/src/types.ts b/langfuse-core/src/types.ts index e9c2882c..d89cbb32 100644 --- a/langfuse-core/src/types.ts +++ b/langfuse-core/src/types.ts @@ -113,6 +113,10 @@ export type GetLangfuseDatasetParams = FixTypes< export type GetLangfuseDatasetResponse = FixTypes< paths["/api/public/v2/datasets/{datasetName}"]["get"]["responses"]["200"]["content"]["application/json"] >; +export type GetLangfuseDatasetsQuery = FixTypes; +export type GetLangfuseDatasetsResponse = FixTypes< + paths["/api/public/v2/datasets"]["get"]["responses"]["200"]["content"]["application/json"] +>; export type GetLangfuseDatasetItemsQuery = paths["/api/public/dataset-items"]["get"]["parameters"]["query"]; export type GetLangfuseDatasetItemsResponse = FixTypes< paths["/api/public/dataset-items"]["get"]["responses"]["200"]["content"]["application/json"] From a708d0819660d0febb6b6b15b6f251ec96ba0c81 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:58:45 +0200 Subject: [PATCH 15/47] refactor: Remove unnecessary code in Langfuse Node.js integration test --- integration-test/langfuse-integration-node.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 5aa672d5..94905db0 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -728,7 +728,6 @@ describe("Langfuse Node.js", () => { const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); - expect(datasetItems.data).toHaveLength(2); expect(datasetItems.data[0]).toContainEqual(expect.objectContaining({ datasetName: datasetName })); expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); From 29c611ad5a280936f42cc90cb5979ca446b767d9 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 18:04:21 +0200 Subject: [PATCH 16/47] refactor: Update Langfuse integration test to use v2 API endpoint for fetching prompts --- .../langfuse-integration-node.spec.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 94905db0..6e79fb3d 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -728,12 +728,38 @@ describe("Langfuse Node.js", () => { const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); - expect(datasetItems.data[0]).toContainEqual(expect.objectContaining({ datasetName: datasetName })); + expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); expect(datasetItems.data[0].metadata).toEqual({ model: "llama3" }); + expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[1].input).toEqual({ text: "goodbye world" }); expect(datasetItems.data[1].expectedOutput).toEqual({ text: "goodbye world" }); expect(datasetItems.data[1].metadata).toEqual({ model: "llama4" }); }); + + it("create and fetch datasets", async () => { + const datasetName1 = utils.generateUUID(); + const datasetName2 = utils.generateUUID(); + const datasetName3 = utils.generateUUID(); + + // Create multiple datasets + langfuse.createDataset({ + name: datasetName1, + description: "My first dataset", + }); + langfuse.createDataset({ + name: datasetName2, + description: "My second dataset", + }); + langfuse.createDataset({ + name: datasetName3, + description: "My third dataset", + }); + + const datasets = await langfuse.getDatasets(); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName1 })); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName2 })); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName3 })); + }); }); From 14e8256a29cd6167458fe3f44c8f5d4fbaad9e22 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 18:12:17 +0200 Subject: [PATCH 17/47] refactor: Update getDatasets method in LangfuseCore to accept optional query parameters --- langfuse-core/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index f583c27b..c3c3ade3 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -898,9 +898,9 @@ export abstract class LangfuseCore extends LangfuseCoreStateless { return this; } - async getDatasets(body: GetLangfuseDatasetsQuery): Promise { + async getDatasets(query?: GetLangfuseDatasetsQuery): Promise { return this.fetch( - `${this.baseUrl}/api/public/datasets?${encodeQueryParams(body)}`, + `${this.baseUrl}/api/public/datasets?${encodeQueryParams(query)}`, this._getFetchOptions({ method: "GET" }) ).then((res) => res.json()); } From dd15bfdf710cf2af9b629d3ba247189e4ba1744f Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 18:26:00 +0200 Subject: [PATCH 18/47] refactor: Update Langfuse integration test to use getDatasetItems method without specifying a dataset name --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 6e79fb3d..76a5aea5 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -726,7 +726,7 @@ describe("Langfuse Node.js", () => { }); await langfuse.flushAsync(); - const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); + const datasetItems = await langfuse.getDatasetItems(); expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); From 00f6b3e0da45b640bb72b54d4ddb672a4fa5232e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 18:29:59 +0200 Subject: [PATCH 19/47] refactor: Update getDatasetItems method in LangfuseCoreStateless class to accept optional query parameters --- langfuse-core/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index c3c3ade3..820c6c4a 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -430,7 +430,7 @@ abstract class LangfuseCoreStateless { ); } - async getDatasetItems(query: GetLangfuseDatasetItemsQuery): Promise { + async getDatasetItems(query?: GetLangfuseDatasetItemsQuery): Promise { return this.fetch( `${this.baseUrl}/api/public/dataset-items?${encodeQueryParams(query)}`, this._getFetchOptions({ method: "GET" }) From e29019af717b9a457fe46f6af8dc6ef4e9f37365 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 18:45:25 +0200 Subject: [PATCH 20/47] refactor: Update Langfuse integration test to use getDatasetItems method with datasetName parameter --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 76a5aea5..58136d88 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -726,8 +726,8 @@ describe("Langfuse Node.js", () => { }); await langfuse.flushAsync(); - const datasetItems = await langfuse.getDatasetItems(); - + const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); + expect(datasetItems.meta["totalItems"]).toEqual(2); expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); From 4029b73fafc6dda7a12786ab5579409c61a2780a Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 19:20:42 +0200 Subject: [PATCH 21/47] refactor: Update Langfuse integration test to use getDatasetItems method with datasetName parameter --- .../langfuse-integration-node.spec.ts | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 58136d88..505f27c5 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -702,40 +702,16 @@ describe("Langfuse Node.js", () => { }); langfuse.createDatasetItem({ datasetName: datasetName, - input: { - text: "hello world", - }, - expectedOutput: { - text: "hello world", - }, - metadata: { - model: "llama3", - }, }); langfuse.createDatasetItem({ datasetName: datasetName, - input: { - text: "goodbye world", - }, - expectedOutput: { - text: "goodbye world", - }, - metadata: { - model: "llama4", - }, }); await langfuse.flushAsync(); const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); expect(datasetItems.meta["totalItems"]).toEqual(2); expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); - expect(datasetItems.data[0].input).toEqual({ text: "hello world" }); - expect(datasetItems.data[0].expectedOutput).toEqual({ text: "hello world" }); - expect(datasetItems.data[0].metadata).toEqual({ model: "llama3" }); expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); - expect(datasetItems.data[1].input).toEqual({ text: "goodbye world" }); - expect(datasetItems.data[1].expectedOutput).toEqual({ text: "goodbye world" }); - expect(datasetItems.data[1].metadata).toEqual({ model: "llama4" }); }); it("create and fetch datasets", async () => { From 961c1b133ef536cc9bc9f6b653b41e92c54af0c2 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 19:26:44 +0200 Subject: [PATCH 22/47] refactor: Update Langfuse integration test to use getDatasetItems method with datasetName parameter --- .../langfuse-integration-datasets.spec.ts | 50 +++++++++++++++++++ .../langfuse-integration-node.spec.ts | 50 ------------------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/integration-test/langfuse-integration-datasets.spec.ts b/integration-test/langfuse-integration-datasets.spec.ts index 8dc8a811..a779332b 100644 --- a/integration-test/langfuse-integration-datasets.spec.ts +++ b/integration-test/langfuse-integration-datasets.spec.ts @@ -1,4 +1,5 @@ // uses the compiled node.js version, run yarn build after making changes to the SDKs +import { utils } from "../langfuse-core/src"; import Langfuse from "../langfuse-node"; describe("Langfuse Node.js", () => { @@ -324,4 +325,53 @@ describe("Langfuse Node.js", () => { }); }, 10000); }); + it("create and fetch dataset items", async () => { + const datasetName = utils.generateUUID(); + langfuse.createDataset({ + name: datasetName, + description: "My first dataset", + metadata: { + author: "Alice", + date: "2022-01-01", + type: "benchmark", + }, + }); + langfuse.createDatasetItem({ + datasetName: datasetName, + }); + langfuse.createDatasetItem({ + datasetName: datasetName, + }); + await langfuse.flushAsync(); + + const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); + expect(datasetItems.meta["totalItems"]).toEqual(2); + expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); + expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); + }); + + it("create and fetch datasets", async () => { + const datasetName1 = utils.generateUUID(); + const datasetName2 = utils.generateUUID(); + const datasetName3 = utils.generateUUID(); + + // Create multiple datasets + langfuse.createDataset({ + name: datasetName1, + description: "My first dataset", + }); + langfuse.createDataset({ + name: datasetName2, + description: "My second dataset", + }); + langfuse.createDataset({ + name: datasetName3, + description: "My third dataset", + }); + + const datasets = await langfuse.getDatasets(); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName1 })); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName2 })); + expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName3 })); + }); }); diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 505f27c5..15393fbb 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -688,54 +688,4 @@ describe("Langfuse Node.js", () => { expect(prompt1Version2.prompt).toEqual("This is prompt 1 version 2"); expect(prompt1Version3.prompt).toEqual("This is prompt 1 version 3"); }); - - it("create and fetch dataset items", async () => { - const datasetName = utils.generateUUID(); - langfuse.createDataset({ - name: datasetName, - description: "My first dataset", - metadata: { - author: "Alice", - date: "2022-01-01", - type: "benchmark", - }, - }); - langfuse.createDatasetItem({ - datasetName: datasetName, - }); - langfuse.createDatasetItem({ - datasetName: datasetName, - }); - await langfuse.flushAsync(); - - const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); - expect(datasetItems.meta["totalItems"]).toEqual(2); - expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); - expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); - }); - - it("create and fetch datasets", async () => { - const datasetName1 = utils.generateUUID(); - const datasetName2 = utils.generateUUID(); - const datasetName3 = utils.generateUUID(); - - // Create multiple datasets - langfuse.createDataset({ - name: datasetName1, - description: "My first dataset", - }); - langfuse.createDataset({ - name: datasetName2, - description: "My second dataset", - }); - langfuse.createDataset({ - name: datasetName3, - description: "My third dataset", - }); - - const datasets = await langfuse.getDatasets(); - expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName1 })); - expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName2 })); - expect(datasets.data).toContainEqual(expect.objectContaining({ name: datasetName3 })); - }); }); From 54873ba811ab61b504e72d49bd79bb056401f46b Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 19:42:29 +0200 Subject: [PATCH 23/47] refactor: Update Langfuse integration test to use getDatasetItems method with datasetName parameter --- integration-test/langfuse-integration-datasets.spec.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-test/langfuse-integration-datasets.spec.ts b/integration-test/langfuse-integration-datasets.spec.ts index a779332b..9627b231 100644 --- a/integration-test/langfuse-integration-datasets.spec.ts +++ b/integration-test/langfuse-integration-datasets.spec.ts @@ -345,6 +345,9 @@ describe("Langfuse Node.js", () => { await langfuse.flushAsync(); const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); + + console.log("Dataset Items:", datasetItems); + expect(datasetItems.meta["totalItems"]).toEqual(2); expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); From 65cf038ff82e3eb3bae24a008f238d5873b5d3c1 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 19:51:40 +0200 Subject: [PATCH 24/47] refactor: Update Langfuse integration test to use getDatasetItems method with datasetName parameter --- .../langfuse-integration-datasets.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/integration-test/langfuse-integration-datasets.spec.ts b/integration-test/langfuse-integration-datasets.spec.ts index 9627b231..e45c8b6b 100644 --- a/integration-test/langfuse-integration-datasets.spec.ts +++ b/integration-test/langfuse-integration-datasets.spec.ts @@ -338,10 +338,29 @@ describe("Langfuse Node.js", () => { }); langfuse.createDatasetItem({ datasetName: datasetName, + input: { + text: "hello world", + }, + expectedOutput: { + text: "hello world", + }, + metadata: { + model: "llama3", + }, }); langfuse.createDatasetItem({ datasetName: datasetName, + input: { + text: "goodbye world", + }, + expectedOutput: { + text: "goodbye world", + }, + metadata: { + model: "llama4", + }, }); + await langfuse.flushAsync(); const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); From 83bacd2029df3bd212a70d9d1008b9f586e84926 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 13:33:10 +0200 Subject: [PATCH 25/47] fix: remove console.log --- integration-test/langfuse-integration-datasets.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-test/langfuse-integration-datasets.spec.ts b/integration-test/langfuse-integration-datasets.spec.ts index e45c8b6b..f6e74e18 100644 --- a/integration-test/langfuse-integration-datasets.spec.ts +++ b/integration-test/langfuse-integration-datasets.spec.ts @@ -365,8 +365,6 @@ describe("Langfuse Node.js", () => { const datasetItems = await langfuse.getDatasetItems({ datasetName: datasetName }); - console.log("Dataset Items:", datasetItems); - expect(datasetItems.meta["totalItems"]).toEqual(2); expect(datasetItems.data[0]).toMatchObject({ datasetName: datasetName }); expect(datasetItems.data[1]).toMatchObject({ datasetName: datasetName }); From fce829700abc68a09e152f75611f1fc239c994b0 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:31:13 +0200 Subject: [PATCH 26/47] feat: Add GetLangfuseScoreResponse type definition --- langfuse-core/src/types.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/langfuse-core/src/types.ts b/langfuse-core/src/types.ts index d89cbb32..53268cc6 100644 --- a/langfuse-core/src/types.ts +++ b/langfuse-core/src/types.ts @@ -99,6 +99,9 @@ export type GetLangfuseSessionsQuery = FixTypes; +export type GetLangfuseScoreResponse = FixTypes< + paths["/api/public/scores/{scoreId}"]["get"]["responses"]["200"]["content"]["application/json"] +>; export type GetLangfuseScoresQuery = FixTypes; export type GetLangfuseScoresResponse = FixTypes< paths["/api/public/scores"]["get"]["responses"]["200"]["content"]["application/json"] From 11c6268a3bb10f39841f364dad16b903cfab0f63 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:31:16 +0200 Subject: [PATCH 27/47] feat: Add fetchScore method to LangfuseCoreStateless class --- langfuse-core/src/index.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 820c6c4a..56901ed9 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -51,6 +51,7 @@ import { type GetLangfusePromptsResponse, type GetLangfuseDatasetsQuery, type GetLangfuseDatasetsResponse, + type GetLangfuseScoreResponse, } from "./types"; import { generateUUID, @@ -344,6 +345,16 @@ abstract class LangfuseCoreStateless { return { data, meta }; } + async fetchScore(scoreId: string): Promise<{ data: GetLangfuseScoreResponse }> { + const res = await this.fetch( + `${this.baseUrl}/api/public/scores?${scoreId}`, + this._getFetchOptions({ method: "GET" }) + ); + // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes + const score = (await res.json()) as GetLangfuseScoreResponse; + return { data: score }; + } + async fetchScores(query?: GetLangfuseScoresQuery): Promise { const res = await this.fetch( `${this.baseUrl}/api/public/scores?${encodeQueryParams(query)}`, From 4bb583c532a9bca4ec1c27e0c5feb6748b908df4 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:31:28 +0200 Subject: [PATCH 28/47] feat: Add create and fetch score by id integration test --- .../langfuse-integration-node.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 15393fbb..96b7e63f 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -552,6 +552,40 @@ describe("Langfuse Node.js", () => { }); }); + it("create and fetch score by id", async () => { + const traceName = utils.generateUUID(); + const trace = langfuse.trace({ + name: traceName, + }); + const score1 = trace.score({ + name: "quality", + value: 1, + comment: "Factually correct", + }); + const score2 = trace.score({ + name: "quality", + value: 0, + comment: "Factually incorrect", + }); + + await langfuse.flushAsync(); + + const fetchedScore1 = await langfuse.fetchScore(score1.id); + expect(fetchedScore1.data).toMatchObject({ + traceId: trace.id, + name: "quality", + value: 1, + comment: "Factually correct", + }); + const fetchedScore2 = await langfuse.fetchScore(score2.id); + expect(fetchedScore2.data).toMatchObject({ + traceId: trace.id, + name: "quality", + value: 0, + comment: "Factually incorrect", + }); + }); + it("create and fetch scores", async () => { const traceName = utils.generateUUID(); const trace = langfuse.trace({ From 730a63a6c873eb809f921406dcfac79dab7a64c5 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:43:04 +0200 Subject: [PATCH 29/47] feat: Add create and fetch scores integration tests --- .../langfuse-integration-node.spec.ts | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 96b7e63f..2947e89a 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -552,23 +552,41 @@ describe("Langfuse Node.js", () => { }); }); + it("create and fetch scores", async () => { + const traceName = utils.generateUUID(); + const trace = langfuse.trace({ + name: traceName, + }); + const score = trace.score({ + name: "quality", + value: 1, + comment: "Factually correct", + }); + await langfuse.flushAsync(); + + const scores = await langfuse.fetchScores({ name: "quality" }); + expect(scores.data).toContainEqual(expect.objectContaining({ traceId: score.id, name: "quality" })); + }); + it("create and fetch score by id", async () => { const traceName = utils.generateUUID(); const trace = langfuse.trace({ name: traceName, }); - const score1 = trace.score({ + trace.score({ name: "quality", value: 1, comment: "Factually correct", }); - const score2 = trace.score({ + trace.score({ name: "quality", value: 0, comment: "Factually incorrect", }); await langfuse.flushAsync(); + const scores = await langfuse.fetchScores(); + const [score2, score1] = scores.data; const fetchedScore1 = await langfuse.fetchScore(score1.id); expect(fetchedScore1.data).toMatchObject({ @@ -586,22 +604,6 @@ describe("Langfuse Node.js", () => { }); }); - it("create and fetch scores", async () => { - const traceName = utils.generateUUID(); - const trace = langfuse.trace({ - name: traceName, - }); - const score = trace.score({ - name: "quality", - value: 1, - comment: "Factually correct", - }); - await langfuse.flushAsync(); - - const scores = await langfuse.fetchScores({ name: "quality" }); - expect(scores.data).toContainEqual(expect.objectContaining({ traceId: score.id, name: "quality" })); - }); - it("create 3 traces with different timestamps and fetch the middle one using to and from timestamp", async () => { const traceName = utils.generateUUID(); const traceParams = [ From 583de1aa3b9ee6ed61deba40b5160e67356723ab Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 17:03:13 +0200 Subject: [PATCH 30/47] refactor: Rearrange score variables in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 2947e89a..477cc326 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -586,7 +586,7 @@ describe("Langfuse Node.js", () => { await langfuse.flushAsync(); const scores = await langfuse.fetchScores(); - const [score2, score1] = scores.data; + const [score1, score2] = scores.data; const fetchedScore1 = await langfuse.fetchScore(score1.id); expect(fetchedScore1.data).toMatchObject({ From c1751cbb6fa1c94253b7c7adf9d701e980798080 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 19:35:32 +0200 Subject: [PATCH 31/47] refactor: Rearrange score variables in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 477cc326..e34a0490 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -586,7 +586,8 @@ describe("Langfuse Node.js", () => { await langfuse.flushAsync(); const scores = await langfuse.fetchScores(); - const [score1, score2] = scores.data; + const score2 = scores.data[0]; + const score1 = scores.data[1]; const fetchedScore1 = await langfuse.fetchScore(score1.id); expect(fetchedScore1.data).toMatchObject({ From f651ffb59194b47a15c8786e9ad32870f21dd9ea Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 19:45:30 +0200 Subject: [PATCH 32/47] refactor: Rearrange score variables in Langfuse integration test --- .../langfuse-integration-node.spec.ts | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index e34a0490..8d087cfa 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -590,19 +590,23 @@ describe("Langfuse Node.js", () => { const score1 = scores.data[1]; const fetchedScore1 = await langfuse.fetchScore(score1.id); - expect(fetchedScore1.data).toMatchObject({ - traceId: trace.id, - name: "quality", - value: 1, - comment: "Factually correct", - }); + expect(fetchedScore1.data).toEqual( + expect.objectContaining({ + traceId: trace.id, + name: "quality", + value: 1, + comment: "Factually correct", + }) + ); const fetchedScore2 = await langfuse.fetchScore(score2.id); - expect(fetchedScore2.data).toMatchObject({ - traceId: trace.id, - name: "quality", - value: 0, - comment: "Factually incorrect", - }); + expect(fetchedScore2.data).toEqual( + expect.objectContaining({ + traceId: trace.id, + name: "quality", + value: 0, + comment: "Factually incorrect", + }) + ); }); it("create 3 traces with different timestamps and fetch the middle one using to and from timestamp", async () => { From c945abda647d037b3ffc3ce23eb8391347adb473 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 20:01:47 +0200 Subject: [PATCH 33/47] feat: Add console log for fetched score in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 8d087cfa..0f4955cd 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -590,6 +590,7 @@ describe("Langfuse Node.js", () => { const score1 = scores.data[1]; const fetchedScore1 = await langfuse.fetchScore(score1.id); + console.log(fetchedScore1.data); expect(fetchedScore1.data).toEqual( expect.objectContaining({ traceId: trace.id, From 304811b85cca82e6c7f93591862bbb374d8d1ec2 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 20:01:55 +0200 Subject: [PATCH 34/47] refactor: Update assertion in Langfuse integration test for fetched score --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 0f4955cd..3221a23c 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -591,7 +591,7 @@ describe("Langfuse Node.js", () => { const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); - expect(fetchedScore1.data).toEqual( + expect(fetchedScore1.data[0]).toEqual( expect.objectContaining({ traceId: trace.id, name: "quality", From d93f985a36cd6f44e3fe39ab8e11cddf0fa7189a Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 20:12:34 +0200 Subject: [PATCH 35/47] refactor: Rearrange score variables in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 3221a23c..8e346576 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -588,6 +588,7 @@ describe("Langfuse Node.js", () => { const scores = await langfuse.fetchScores(); const score2 = scores.data[0]; const score1 = scores.data[1]; + console.log(score1.id, score2.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); @@ -600,7 +601,7 @@ describe("Langfuse Node.js", () => { }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); - expect(fetchedScore2.data).toEqual( + expect(fetchedScore2.data[0]).toEqual( expect.objectContaining({ traceId: trace.id, name: "quality", From bada350be3a8e0175171191432be851d65dd662f Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 20:17:09 +0200 Subject: [PATCH 36/47] refactor: Rearrange score variables and update assertion in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 8e346576..7c8baf00 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -592,7 +592,7 @@ describe("Langfuse Node.js", () => { const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); - expect(fetchedScore1.data[0]).toEqual( + expect(fetchedScore1.data).toEqual( expect.objectContaining({ traceId: trace.id, name: "quality", @@ -601,7 +601,7 @@ describe("Langfuse Node.js", () => { }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); - expect(fetchedScore2.data[0]).toEqual( + expect(fetchedScore2.data).toEqual( expect.objectContaining({ traceId: trace.id, name: "quality", From 2a6ee06a76f3bb041d92c6ab3478352c15e85982 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 20:22:25 +0200 Subject: [PATCH 37/47] refactor: Rearrange score variables and update assertion in Langfuse integration test --- integration-test/langfuse-integration-node.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 7c8baf00..0f4955cd 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -588,7 +588,6 @@ describe("Langfuse Node.js", () => { const scores = await langfuse.fetchScores(); const score2 = scores.data[0]; const score1 = scores.data[1]; - console.log(score1.id, score2.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); From 0729e45dfecc245c439a410f60c5c54f2a7cd8b1 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 12:11:39 +0200 Subject: [PATCH 38/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 0f4955cd..d5afadda 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -586,8 +586,8 @@ describe("Langfuse Node.js", () => { await langfuse.flushAsync(); const scores = await langfuse.fetchScores(); - const score2 = scores.data[0]; - const score1 = scores.data[1]; + const score2 = scores.data[1]; + const score1 = scores.data[0]; const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); From f7a249ed15e0bcd3c9b0685862aaf3bff28b4013 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 12:34:34 +0200 Subject: [PATCH 39/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- .../langfuse-integration-node.spec.ts | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index d5afadda..2bdbc617 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -573,39 +573,38 @@ describe("Langfuse Node.js", () => { const trace = langfuse.trace({ name: traceName, }); + trace.score({ + name: "harmfulness", + value: 0.5, + }); trace.score({ name: "quality", value: 1, comment: "Factually correct", }); trace.score({ - name: "quality", - value: 0, - comment: "Factually incorrect", + name: "relevance", + value: 0.8, + comment: "Mostly relevant", }); await langfuse.flushAsync(); const scores = await langfuse.fetchScores(); + const score1 = scores.data[2]; const score2 = scores.data[1]; - const score1 = scores.data[0]; const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); - expect(fetchedScore1.data).toEqual( - expect.objectContaining({ - traceId: trace.id, - name: "quality", - value: 1, - comment: "Factually correct", - }) + expect(fetchedScore1.data).toContainEqual( + expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); - expect(fetchedScore2.data).toEqual( + expect(fetchedScore2.data).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "quality", - value: 0, - comment: "Factually incorrect", + value: 1, + comment: "Factually correct", }) ); }); From 8b1c5122e9bea0cbe975b793a32fae7df4ff750e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 12:50:54 +0200 Subject: [PATCH 40/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 2bdbc617..23e68b8e 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -592,10 +592,10 @@ describe("Langfuse Node.js", () => { const scores = await langfuse.fetchScores(); const score1 = scores.data[2]; const score2 = scores.data[1]; - + console.log(score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); - expect(fetchedScore1.data).toContainEqual( + expect(fetchedScore1.data[0]).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); From 2cc5fd1ec385919a488bee2e5a2fbe30d9d38cb2 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 13:38:20 +0200 Subject: [PATCH 41/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 23e68b8e..62181bb6 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -595,7 +595,7 @@ describe("Langfuse Node.js", () => { console.log(score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); - expect(fetchedScore1.data[0]).toContainEqual( + expect(fetchedScore1.data).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); From 481a199abf5ce9c9fefce442d44b848a69e03303 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 13:39:19 +0200 Subject: [PATCH 42/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 62181bb6..00f0a1ac 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -592,7 +592,7 @@ describe("Langfuse Node.js", () => { const scores = await langfuse.fetchScores(); const score1 = scores.data[2]; const score2 = scores.data[1]; - console.log(score1.id); + console.log("ScoreId", score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); console.log(fetchedScore1.data); expect(fetchedScore1.data).toContainEqual( From 79365b8dffddfc14a72b02cbcb6681b8f8aaac59 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 13:51:01 +0200 Subject: [PATCH 43/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 2 +- langfuse-core/src/index.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 00f0a1ac..7094dbfb 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -594,7 +594,7 @@ describe("Langfuse Node.js", () => { const score2 = scores.data[1]; console.log("ScoreId", score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); - console.log(fetchedScore1.data); + console.log("ScoreData", fetchedScore1.data); expect(fetchedScore1.data).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index e00e1f69..46398f95 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -362,8 +362,7 @@ abstract class LangfuseCoreStateless { this._getFetchOptions({ method: "GET" }) ); // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes - const { data, meta } = (await res.json()) as GetLangfuseScoresResponse; - return { data, meta }; + return (await res.json()) as GetLangfuseScoresResponse; } async fetchPrompts(query?: GetLangfusePromptsQuery): Promise { From 69881c7653166fb42890c10774b7a1aa06821d48 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 13:59:53 +0200 Subject: [PATCH 44/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- langfuse-core/src/index.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 7094dbfb..dfb9a499 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -594,8 +594,8 @@ describe("Langfuse Node.js", () => { const score2 = scores.data[1]; console.log("ScoreId", score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); - console.log("ScoreData", fetchedScore1.data); - expect(fetchedScore1.data).toContainEqual( + console.log("ScoreData", fetchedScore1.data.data); + expect(fetchedScore1.data.data).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index 46398f95..cd98b087 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -351,7 +351,7 @@ abstract class LangfuseCoreStateless { `${this.baseUrl}/api/public/scores?${scoreId}`, this._getFetchOptions({ method: "GET" }) ); - // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes + const score = (await res.json()) as GetLangfuseScoreResponse; return { data: score }; } @@ -362,7 +362,8 @@ abstract class LangfuseCoreStateless { this._getFetchOptions({ method: "GET" }) ); // destructure the response into data and meta to be explicit about the shape of the response and add type-warnings in case the API changes - return (await res.json()) as GetLangfuseScoresResponse; + const { data, meta } = (await res.json()) as GetLangfuseScoresResponse; + return { data, meta }; } async fetchPrompts(query?: GetLangfusePromptsQuery): Promise { From 80c7fc66448287f634b990f3ec56009d521e9700 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 14:01:00 +0200 Subject: [PATCH 45/47] Refactor score data retrieval URL in LangfuseCoreStateless class --- langfuse-core/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langfuse-core/src/index.ts b/langfuse-core/src/index.ts index cd98b087..0c4bbf97 100644 --- a/langfuse-core/src/index.ts +++ b/langfuse-core/src/index.ts @@ -348,7 +348,7 @@ abstract class LangfuseCoreStateless { async fetchScore(scoreId: string): Promise<{ data: GetLangfuseScoreResponse }> { const res = await this.fetch( - `${this.baseUrl}/api/public/scores?${scoreId}`, + `${this.baseUrl}/api/public/scores/${scoreId}`, this._getFetchOptions({ method: "GET" }) ); From 9006b3ddb5473c6dfd7eeb6b526c7e87e34340c0 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 14:01:33 +0200 Subject: [PATCH 46/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- integration-test/langfuse-integration-node.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index dfb9a499..7094dbfb 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -594,8 +594,8 @@ describe("Langfuse Node.js", () => { const score2 = scores.data[1]; console.log("ScoreId", score1.id); const fetchedScore1 = await langfuse.fetchScore(score1.id); - console.log("ScoreData", fetchedScore1.data.data); - expect(fetchedScore1.data.data).toContainEqual( + console.log("ScoreData", fetchedScore1.data); + expect(fetchedScore1.data).toContainEqual( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); From ca3c24d8b4f0cd2b61b003dca4c8a2fbeb3f0737 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 14:16:08 +0200 Subject: [PATCH 47/47] Refactor score data retrieval in langfuse-integration-node.spec.ts --- .../langfuse-integration-node.spec.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/integration-test/langfuse-integration-node.spec.ts b/integration-test/langfuse-integration-node.spec.ts index 7094dbfb..2f02d72a 100644 --- a/integration-test/langfuse-integration-node.spec.ts +++ b/integration-test/langfuse-integration-node.spec.ts @@ -592,14 +592,13 @@ describe("Langfuse Node.js", () => { const scores = await langfuse.fetchScores(); const score1 = scores.data[2]; const score2 = scores.data[1]; - console.log("ScoreId", score1.id); + const score3 = scores.data[0]; const fetchedScore1 = await langfuse.fetchScore(score1.id); - console.log("ScoreData", fetchedScore1.data); - expect(fetchedScore1.data).toContainEqual( + expect(fetchedScore1.data).toMatchObject( expect.objectContaining({ traceId: trace.id, name: "harmfulness", value: 0.5 }) ); const fetchedScore2 = await langfuse.fetchScore(score2.id); - expect(fetchedScore2.data).toContainEqual( + expect(fetchedScore2.data).toMatchObject( expect.objectContaining({ traceId: trace.id, name: "quality", @@ -607,6 +606,15 @@ describe("Langfuse Node.js", () => { comment: "Factually correct", }) ); + const fetchedScore3 = await langfuse.fetchScore(score3.id); + expect(fetchedScore3.data).toMatchObject( + expect.objectContaining({ + traceId: trace.id, + name: "relevance", + value: 0.8, + comment: "Mostly relevant", + }) + ); }); it("create 3 traces with different timestamps and fetch the middle one using to and from timestamp", async () => {