init vec db

package.json

@@ -31,15 +31,19 @@
         "@langchain/ollama": "^0.1.4",
         "@langchain/openai": "0.3.16",
         "agent-twitter-client": "0.0.18",
+        "better-sqlite3": "^11.8.0",
         "dotenv": "^16.3.1",
         "ethers": "^6.13.4",
+        "sqlite3": "^5.1.7",
+        "vectorlite": "^0.2.0",
         "winston": "^3.11.0",
         "zod": "^3.22.4",
         "zod-to-json-schema": "^3.24.1"
     },
     "devDependencies": {
         "@eslint/js": "^9.18.0",
         "@tsconfig/node20": "^20.1.4",
+        "@types/better-sqlite3": "^7.6.12",
         "@types/jest": "^29.5.12",
         "@types/js-yaml": "^4.0.9",
         "@types/node": "22.10.0",

src/vecdb/index.ts

@@ -0,0 +1,63 @@
+import Database from "better-sqlite3";
+import vectorlite from "vectorlite";
+import fs from "fs";
+
+const extensionPath = vectorlite.vectorlitePath();
+console.log("Vectorlite extension path:", extensionPath);
+
+const memoryDb = new Database(":memory:");
+
+memoryDb.loadExtension(extensionPath);
+console.log("Vectorlite loaded successfully!");
+
+memoryDb.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS my_table USING vectorlite(
+        my_embedding float32[3], 
+        hnsw(max_elements=100)
+    )
+`);
+
+const diskDbPath = "mydatabase.db";
+
+const diskDbExists = fs.existsSync(diskDbPath);
+
+if (diskDbExists) {
+    console.log("Loading database from disk...");
+    const diskDb = new Database(diskDbPath, { readonly: true });
+    const backup = memoryDb.backup(diskDbPath);
+    await backup;
+    console.log("Database loaded into memory.");
+} else {
+    console.log("No existing disk database found. Starting fresh.");
+}
+
+const insertStmt = memoryDb.prepare(`
+    INSERT INTO my_table (rowid, my_embedding) 
+    VALUES (?, ?)
+`);
+
+insertStmt.run(1, Buffer.from(new Float32Array([1, 2, 3]).buffer));
+insertStmt.run(2, Buffer.from(new Float32Array([4, 5, 6]).buffer));
+insertStmt.run(3, Buffer.from(new Float32Array([7, 8, 9]).buffer));
+
+console.log("Vectors inserted into memory!");
+
+const queryStmt = memoryDb.prepare(`
+    SELECT rowid, distance 
+    FROM my_table 
+    WHERE knn_search(my_embedding, knn_param(?, 2))
+`);
+
+const queryResult = queryStmt.all(Buffer.from(new Float32Array([3, 4, 5]).buffer));
+
+console.log("Nearest neighbors:", queryResult);
+
+console.log("Saving database to disk...");
+const diskDb = new Database(diskDbPath);
+const backup = memoryDb.backup(diskDbPath);
+await backup;
+console.log("Database saved to disk.");
+
+memoryDb.close();
+diskDb.close();
+console.log("Databases closed.");

src/vecdb/types/vectorlite.d.ts

@@ -0,0 +1,3 @@
+declare module 'vectorlite' {
+    export function vectorlitePath(): string;
+}