refactor: files api

consts/api.ts

@@ -24,5 +24,13 @@ export const STEP_KEY = "step";
 export const STEP_OBJECT = "thread.run.step";
 export const STEP_PREFIX = "step";
 
+export const FILE_KEY = "file";
+export const FILE_OBJECT = "file";
+export const FILE_PREFIX = "file";
+
 // 10 minutes, unit: second
 export const RUN_EXPIRED_DURATION = 10 * 60;
+
+// files api
+export const DEFAULT_FILE_DIR = "/tmp/assistant";
+export const DEFAULT_ORG_FILES_SIZE_MAX = 100_000_000_000; // 100GB

consts/api_test.ts

@@ -18,7 +18,12 @@ import {
   STEP_KEY,
   STEP_OBJECT,
   STEP_PREFIX,
+  FILE_KEY,
+  FILE_OBJECT,
+  FILE_PREFIX,
   RUN_EXPIRED_DURATION,
+  DEFAULT_FILE_DIR,
+  DEFAULT_ORG_FILES_SIZE_MAX,
 } from "$/consts/api.ts";
 
 describe("HTTP Headers", () => {
@@ -92,7 +97,29 @@ describe("Model consts", () => {
     assertEquals(STEP_PREFIX, "step");
   });
 
+  it("has FILE_KEY const", () => {
+    assertEquals(FILE_KEY, "file");
+  });
+
+  it("has FILE_OBJECT const", () => {
+    assertEquals(FILE_OBJECT, "file");
+  });
+
+  it("has FILE_PREFIX const", () => {
+    assertEquals(FILE_PREFIX, "file");
+  });
+});
+
+describe("API consts", () => {
   it("has RUN_EXPIRED_DURATION const", () => {
     assertEquals(RUN_EXPIRED_DURATION, 600);
   });
+
+  it("has DEFAULT_FILE_DIR const", () => {
+    assertEquals(DEFAULT_FILE_DIR, "/tmp/assistant");
+  });
+
+  it("has DEFAULT_ORG_FILES_SIZE_MAX const", () => {
+    assertEquals(DEFAULT_ORG_FILES_SIZE_MAX, 100_000_000_000);
+  });
 });

consts/envs.ts

@@ -15,3 +15,7 @@ export const MODEL_KNOWLEDGE_CUTOFF = "MODEL_KNOWLEDGE_CUTOFF";
 export const ANTHROPIC_API_URL = "ANTHROPIC_API_URL";
 export const ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY";
 export const ANTHROPIC_VERSION = "ANTHROPIC_VERSION";
+
+// file
+export const FILE_DIR = "FILE_DIR";
+export const ORG_FILES_SIZE_MAX = "ORG_FILES_SIZE_MAX";

consts/envs_test.ts

@@ -9,6 +9,8 @@ import {
   ANTHROPIC_API_KEY,
   ANTHROPIC_API_URL,
   ANTHROPIC_VERSION,
+  FILE_DIR,
+  ORG_FILES_SIZE_MAX,
 } from "$/consts/envs.ts";
 
 describe("Log variables", () => {
@@ -48,3 +50,13 @@ describe("Anthropic variables", () => {
     assertExists(ANTHROPIC_VERSION);
   });
 });
+
+describe("Files API variables", () => {
+  it("The FILE_DIR const exists", () => {
+    assertExists(FILE_DIR);
+  });
+
+  it("The ORG_FILES_SIZE_MAX const exists", () => {
+    assertExists(ORG_FILES_SIZE_MAX);
+  });
+});

deno.json

@@ -17,7 +17,7 @@
     "$/": "./",
     "$fresh/": "https://deno.land/x/[email protected]/",
     "$std/": "https://deno.land/[email protected]/",
-    "@open-schemas/zod": "jsr:@open-schemas/zod@^0.8.4",
+    "@open-schemas/zod": "jsr:@open-schemas/zod@^0.9.2",
     "@std/assert": "jsr:@std/assert@^0.221.0",
     "@std/log": "jsr:@std/log@^0.221.0",
     "@std/testing": "jsr:@std/testing@^0.221.0",

fresh.gen.ts

@@ -14,6 +14,7 @@ import * as $v1_assistants_assistant_id_ from "./routes/v1/assistants/[assistant
 import * as $v1_assistants_index from "./routes/v1/assistants/index.ts";
 import * as $v1_chat_completions from "./routes/v1/chat/completions.ts";
 import * as $v1_files_file_id_ from "./routes/v1/files/[file_id].ts";
+import * as $v1_files_file_id_content from "./routes/v1/files/[file_id]/content.ts";
 import * as $v1_files_index from "./routes/v1/files/index.ts";
 import * as $v1_models_index from "./routes/v1/models/index.ts";
 import * as $v1_threads_thread_id_ from "./routes/v1/threads/[thread_id].ts";
@@ -43,6 +44,7 @@ const manifest = {
     "./routes/v1/assistants/index.ts": $v1_assistants_index,
     "./routes/v1/chat/completions.ts": $v1_chat_completions,
     "./routes/v1/files/[file_id].ts": $v1_files_file_id_,
+    "./routes/v1/files/[file_id]/content.ts": $v1_files_file_id_content,
     "./routes/v1/files/index.ts": $v1_files_index,
     "./routes/v1/models/index.ts": $v1_models_index,
     "./routes/v1/threads/[thread_id].ts": $v1_threads_thread_id_,

repositories/file.ts

@@ -1,45 +1,31 @@
-import { Repository } from "$/repositories/_repository.ts";
-import type { FileObjectType } from "$/schemas/file.ts";
+import { FileObject } from "@open-schemas/zod/openai";
+import { Repository } from "$/repositories/base.ts";
+import { FILE_KEY, FILE_OBJECT, FILE_PREFIX, ORGANIZATION } from "$/consts/api.ts";
 
-export class FileRepository extends Repository {
-  static idPrefix = "file";
-  static object = "file";
-  static parent = "organization";
-  static self = "file";
+export class FileRepository extends Repository<FileObject> {
+  private static instance: FileRepository;
 
-  static async sumFileSize(org: string) {
-    const files = await this.findAll<FileObjectType>(org);
-    return files.reduce((pre, cur) => pre + cur.bytes, 0);
+  private constructor() {
+    super(FILE_PREFIX, FILE_OBJECT, ORGANIZATION, FILE_KEY);
   }
 
-  // private static genFileSizeKey(org: string) {
-  //   return [this.parent, org, "file_size"];
-  // }
-
-  // static async createWithBytes(
-  //   fields: Partial<FileObjectType>,
-  //   organization: string,
-  // ) {
-  //   const operation = kv.atomic();
-  //   const { value } = await this.create<FileObjectType>(
-  //     fields,
-  //     organization,
-  //     operation,
-  //   );
-  //   operation.sum(
-  //     [this.parent, organization, "file_size"],
-  //     BigInt(value.bytes),
-  //   );
-
-  //   const { ok } = await operation.commit();
-  //   if (!ok) throw new DbCommitError();
-  //   return { value };
-  // }
+  public static getInstance(): FileRepository {
+    if (!FileRepository.instance) {
+      FileRepository.instance = new FileRepository();
+    }
+    return FileRepository.instance;
+  }
 
-  // static async destoryWithBytes(id: string, org: string, bytes: number) {
-  //   const operation = kv.atomic();
-  //   this.destory(id, org, operation);
+  async sumFileSize(org: string) {
+    const files = await this.findAll(org);
+    return files.reduce((pre, cur) => pre + cur.bytes, 0);
+  }
 
-  //   operation.sum(this.genFileSizeKey(org), -BigInt(bytes));
-  // }
+  async findByPurpose(organization: string, purpose?: string) {
+    const files = await this.findAll(organization);
+    if (purpose) {
+      return files.filter((f) => f.purpose === purpose);
+    }
+    return files;
+  }
 }

routes/v1/files/[file_id].ts

@@ -1,20 +1,19 @@
 import { FreshContext, Handlers } from "$fresh/server.ts";
 import { FileRepository } from "$/repositories/file.ts";
-import { DeleteFileResponse } from "openai_schemas";
-import type { FileObjectType } from "$/schemas/file.ts";
+import { FileObject, DeleteFileResponse } from "@open-schemas/zod/openai";
 
 const getIDs = (ctx: FreshContext) => ({
   id: ctx.params.file_id as string,
   parentId: ctx.state.organization as string,
 });
 
-async function getFile(ctx: FreshContext) {
+export async function getFile(ctx: FreshContext) {
   const { id, parentId } = getIDs(ctx);
 
-  return await FileRepository.findById<FileObjectType>(id, parentId);
+  return await FileRepository.getInstance().findById(id, parentId);
 }
 
-export const handler: Handlers<FileObjectType | null> = {
+export const handler: Handlers<FileObject | null> = {
   async GET(_req, ctx: FreshContext) {
     return Response.json(await getFile(ctx));
   },
@@ -23,7 +22,7 @@ export const handler: Handlers<FileObjectType | null> = {
     await getFile(ctx);
     const { id, parentId } = getIDs(ctx);
 
-    await FileRepository.destory(id, parentId);
+    await FileRepository.getInstance().destory(id, parentId);
     return Response.json(DeleteFileResponse.parse({ id }));
   },
 };

routes/v1/files/[file_id]/content.ts

@@ -0,0 +1,14 @@
+import { FreshContext, Handlers } from "$fresh/server.ts";
+import { FileObject } from "@open-schemas/zod/openai";
+import { getFile } from "$/routes/v1/files/[file_id].ts";
+import { getFileDir } from "$/utils/file.ts";
+
+export const handler: Handlers<FileObject | null> = {
+  async GET(_req, ctx: FreshContext) {
+    const fileObject = await getFile(ctx);
+    const organization = ctx.state.organization as string;
+    const dirPath = `${getFileDir()}/${organization}`;
+    const file = await Deno.open(`${dirPath}/${fileObject.id}`, { read: true });
+    return new Response(file.readable);
+  },
+};

routes/v1/files/index.ts

@@ -1,49 +1,29 @@
 import { FreshContext, Handlers } from "$fresh/server.ts";
-import { z } from "zod";
-import { pagableSchema, sortSchema } from "$/repositories/_repository.ts";
+import { FileObject, UploadFileRequest } from "@open-schemas/zod/openai";
 import { FileRepository } from "$/repositories/file.ts";
 import * as log from "$std/log/mod.ts";
-import { ensureDir } from "$/utils/file.ts";
-import { getFileDir } from "$/utils/file.ts";
-import type { FileObjectType } from "$/schemas/file.ts";
+import { ensureDir, getFileDir, getOrgFilesSizeMax } from "$/utils/file.ts";
 import { UnprocessableContent } from "$/utils/errors.ts";
-import { getOrgFilesSizeMax } from "$/utils/file.ts";
+import { kv } from "$/repositories/_repository.ts";
 
-export const CreateFileRequest = z.object({
-  file: z.object({
-    name: z.string(),
-    size: z
-      .number({
-        description: "The size of individual files can be a maximum of 512 MB.",
-      })
-      .int()
-      .min(1)
-      .max(512_000_000),
-  }),
-  purpose: z.enum(["fine-tune", "assistants"]),
-});
-
-export const handler: Handlers<FileObjectType | null> = {
+export const handler: Handlers<FileObject | null> = {
   async GET(_req: Request, ctx: FreshContext) {
     const params = Object.fromEntries(ctx.url.searchParams);
     const organization = ctx.state.organization as string;
 
-    const page = await FileRepository.findAllByPage<FileObjectType>(
-      organization,
-      pagableSchema.parse(params),
-      sortSchema.parse(params),
-    );
+    const data = await FileRepository.getInstance().findByPurpose(organization, params["purpose"]);
 
-    return Response.json(page);
+    return Response.json({ object: "list", data });
   },
 
   async POST(req: Request, ctx: FreshContext) {
     const form = await req.formData();
     const file = form.get("file") as File;
-    const fields = CreateFileRequest.parse({
+    const fields = UploadFileRequest.parse({
       file: file && {
         name: file.name,
         size: file.size,
+        type: file.type,
       },
       purpose: form.get("purpose") as string,
     });
@@ -54,31 +34,37 @@ export const handler: Handlers<FileObjectType | null> = {
       });
     }
     const organization = ctx.state.organization as string;
-    const allFileSize = await FileRepository.sumFileSize(organization);
+    const fileRepository = FileRepository.getInstance();
+    const allFileSize = await fileRepository.sumFileSize(organization);
     const max = getOrgFilesSizeMax();
     if (allFileSize + file.size > max) {
       throw new UnprocessableContent(undefined, {
         cause: `The size of all the files uploaded by one organization can be up to ${max} Bytes. Current is ${allFileSize} Bytes.`,
       });
     }
 
-    const { value } = await FileRepository.create<FileObjectType>(
+    const operation = kv.atomic();
+    const fileObject = await fileRepository.create(
       {
         purpose: fields.purpose,
         bytes: fields.file.size,
         filename: fields.file.name,
+        filetype: fields.file.type,
       },
       organization,
+      operation,
     );
-    log.debug(`value: ${JSON.stringify(value)}`);
+    log.debug(`value: ${JSON.stringify(fileObject)}`);
 
     const dirPath = `${getFileDir()}/${organization}`;
     await ensureDir(dirPath);
-    Deno.writeFile(`${dirPath}/${value.id}`, file.stream(), {
+    Deno.writeFile(`${dirPath}/${fileObject.id}`, file.stream(), {
       create: true,
     });
 
-    return Response.json(value, {
+    await operation.commit();
+
+    return Response.json(fileObject, {
       status: 201,
     });
   },