cleaner semantic search implementation

electron-src/constants.ts

@@ -1,14 +1,10 @@
 import path from "path";
 import { app, nativeImage } from "electron";
-import { createWriteStream } from "fs";
 import isDev from "electron-is-dev";
 
 const APP_PATH = app.getAppPath();
 export const RESOURCES_PATH = APP_PATH.endsWith("app.asar")
   ? path.dirname(APP_PATH) // If we're bundled, resources are above the bundle
   : APP_PATH.split("build")[0]; // Otherwise everything is in the root of the app
-export const LOGS_PATH = app.getPath("logs");
-export const logPath = path.join(LOGS_PATH, `run-${new Date().toISOString()}.log`);
-export const logStream = createWriteStream(logPath);
 export const mainAppIconDevPng = nativeImage.createFromPath(path.join(RESOURCES_PATH, "assets", "icon.png"));
 export const debugLoggingEnabled = isDev && process.env.DEBUG_LOGGING === "true";

electron-src/data/base-database.ts

@@ -93,7 +93,7 @@ export class BaseDatabase<T> {
       this.dbWriter = db;
       return true;
     } catch (e) {
-      console.error(e);
+      logger.error(e);
       return false;
     } finally {
       this.isSettingUpDb = false;

electron-src/data/embeddings-database.ts

@@ -26,7 +26,7 @@ export class EmbeddingsDatabase extends BaseDatabase<EmbeddingsDb> {
   };
   loadVectorsIntoMemory = async () => {
     if (this.embeddingsCache.length) {
-      return this.embeddingsCache;
+      return;
     }
     await this.initialize();
     const result = await this.db.selectFrom("embeddings").selectAll().execute();
@@ -47,6 +47,10 @@ export class EmbeddingsDatabase extends BaseDatabase<EmbeddingsDb> {
     await this.loadVectorsIntoMemory();
     return this.embeddingsCache;
   };
+
+  embeddingsCacheSize = () => {
+    return this.embeddingsCache.length;
+  };
   getEmbeddingByText = async (text: string) => {
     await this.initialize();
     const result = await this.db.selectFrom("embeddings").where("text", "=", text).selectAll().executeTakeFirst();

electron-src/hooks/notarize.ts

@@ -12,11 +12,11 @@ if (DEV_MODE || process.platform !== "darwin") {
   console.log("Skipping notarization - not building for Mac");
   process.exit(0);
 }
-//
-// if (!process.env.APPLE_ID) {
-//   console.log("Skipping notarization - no apple id");
-//   process.exit(0);
-// }
+
+if (!process.env.APPLE_ID) {
+  console.log("Skipping notarization - no apple id");
+  process.exit(0);
+}
 
 console.log("Notarizing...");
 

electron-src/index.ts

@@ -1,10 +1,12 @@
+import "./utils/dns-cache";
 import { app, Menu, nativeTheme, protocol, shell } from "electron";
+addFlags(app);
+
 // Global imports to monkeypatch/polyfill/register
 import "./semantic-search/semantic-search";
 import "./ipc/ipc";
 import "./options";
 import "./ipc/ipc-onboarding";
-import "./utils/dns-cache";
 // normal imports
 import type { CustomScheme } from "electron";
 import { addFlags } from "./utils/flags";
@@ -14,14 +16,14 @@ import registerContextMenu from "electron-context-menu";
 import { getMenu } from "./window/menu";
 
 import "better-sqlite3";
-import { logPath, logStream, mainAppIconDevPng } from "./constants";
-import logger from "./utils/logger";
+import { mainAppIconDevPng } from "./constants";
+import logger, { fileLogFormat } from "./utils/logger";
 import { setupRouteHandlers } from "./utils/routes";
 import { DESKTOP_VERSION } from "./versions";
 import { autoUpdater } from "electron-updater";
 import dbWorker from "./workers/database-worker";
 import winston from "winston";
-addFlags(app);
+import { logPath, logStream } from "./logs";
 
 registerContextMenu({
   showSaveImageAs: true,
@@ -40,10 +42,10 @@ let errorTries = 0;
 const MAX_ERROR_TRIES = 5;
 
 const amMainInstance = app.requestSingleInstanceLock();
-
-logger.transports.push(
+logger.add(
   new winston.transports.Stream({
     stream: logStream,
+    format: fileLogFormat,
   }),
 );
 logger.info(`Starting logging to ${logPath}`);

electron-src/logs.ts

@@ -0,0 +1,15 @@
+import fs, { createWriteStream } from "fs";
+import path from "path";
+import { threadId } from "node:worker_threads";
+import os from "os";
+import { appPath } from "./versions";
+
+const isDev = process.env.NODE_ENV !== "production";
+const Logs = path.join(os.homedir(), "Library", "Logs", appPath);
+// ensure directory exists recursively
+if (!fs.existsSync(Logs)) {
+  fs.mkdirSync(Logs, { recursive: true });
+}
+const workerPrefix = threadId ? `worker-${threadId}-` : "";
+export const logPath = path.join(Logs, `${isDev ? "dev-" : ""}${workerPrefix}run-${new Date().toISOString()}.log`);
+export const logStream = createWriteStream(logPath);

electron-src/utils/buffer.ts

@@ -18,7 +18,6 @@ export const decodeMessageBuffer = async (buffer: Buffer | Uint8Array | undefine
       return unarchiver.decodeAll();
     }
   } catch (e) {
-    console.error(e);
     // ignore
   }
   return buffer;

electron-src/utils/logger.ts

@@ -5,7 +5,7 @@ const { combine, timestamp, printf, colorize, errors, json, splat } = winston.fo
 const ts = timestamp({
   format: "YYYY-MM-DD HH:mm:ss",
 });
-const print = printf((info) => {
+export const print = printf((info) => {
   if (typeof info.message === "object") {
     info.message = JSON.stringify(info.message);
   }
@@ -17,10 +17,10 @@ const print = printf((info) => {
 });
 
 const localFormat = combine(ts, colorize(), splat(), errors({ stack: true }), print);
+export const fileLogFormat = combine(ts, splat(), errors({ stack: true }), print);
 
 export const logger = winston.createLogger({
   level: "debug",
-  format: localFormat,
   transports: [
     new winston.transports.Console({
       format: localFormat,

electron-src/utils/util.ts

@@ -3,6 +3,7 @@ import { app, dialog } from "electron";
 import { windows } from "../index";
 import { createMainWindow } from "../window/main-window";
 import { installExtension, REACT_DEVELOPER_TOOLS } from "electron-extension-installer";
+import logger from "./logger";
 
 export interface Deferred<T> {
   resolve: (arg: T) => void;
@@ -28,7 +29,7 @@ export function showErrorAlert(title: string, body: string, logStream?: WriteStr
   if (logStream) {
     logStream.write(`ALERT: ${title}: ${body}\n`);
   }
-  console.warn(`${title}: ${body}`);
+  logger.warn(`${title}: ${body}`);
   dialog.showErrorBox(title, body);
 }
 export const showApp = () => {
@@ -46,7 +47,7 @@ export const withRetries = async (fn: () => Promise<void>, MAX_ERROR_TRIES = 5)
       await fn();
       return;
     } catch (e) {
-      console.log(e);
+      logger.error(e);
       if (i == MAX_ERROR_TRIES - 1) {
         throw e;
       }

electron-src/window/main-window.ts

@@ -3,14 +3,15 @@ import { app, screen } from "electron";
 import windowStateKeeper from "electron-window-state";
 import createWindow from "./create-window";
 import { join } from "path";
-import { logStream, mainAppIconDevPng } from "../constants";
+import { mainAppIconDevPng } from "../constants";
 import isDev from "electron-is-dev";
 import { format } from "url";
 import { showErrorAlert, withRetries } from "../utils/util";
 import prepareNext from "../utils/next-helper";
 import logger from "../utils/logger";
 import { windows } from "../index";
 import { addWebRequestToSession } from "../utils/routes";
+import { logStream } from "../logs";
 
 const setupNext = async () => {
   try {

electron-src/window/menu.ts

@@ -6,7 +6,8 @@ import { showApp } from "../utils/util";
 import { requestContactsPerms, requestFullDiskAccess } from "../ipc/ipc-onboarding";
 import { clearSkipContactsPermsCheck } from "../options";
 import { copyDbAtPath, copyLatestDb } from "../data/db-file-utils";
-import { logPath } from "../constants";
+import logger from "../utils/logger";
+import { logPath } from "../logs";
 
 export const getMenu = () => {
   const menuTemplate: MenuItemConstructorOptions[] = [
@@ -143,7 +144,7 @@ export const getMenu = () => {
         label: "Quit",
         accelerator: "CmdOrCtrl+Q",
         click: () => {
-          console.log("Cmd + Q is pressed");
+          logger.info("Cmd + Q is pressed");
           windows.forEach((win) => win.close());
           app.quit();
         },

electron-src/workers/database-worker.ts

@@ -8,6 +8,7 @@ import { join } from "path";
 import logger from "../utils/logger";
 import type { EmbeddingsDatabase } from "../data/embeddings-database";
 import embeddingsDb from "../data/embeddings-database";
+import type { ThreadsWorkerOptions } from "threads/dist/types/master";
 
 type WorkerType<T> = {
   [P in keyof T]: T[P] extends (...args: infer A) => infer R ? (...args: A) => Promise<R> : never;
@@ -18,25 +19,22 @@ class DbWorker {
   embeddingsWorker!: WorkerType<EmbeddingsDatabase> | EmbeddingsDatabase;
 
   startWorker = async () => {
-    const path = isDev ? "workers/worker.js" : join("..", "..", "..", "app.asar.unpacked", "worker.js");
-    const embeddingWorkerPath = isDev
-      ? "workers/embeddings-worker.js"
-      : join("..", "..", "..", "app.asar.unpacked", "embeddings-worker.js");
+    if (isDev) {
+      this.worker = db;
+      this.embeddingsWorker = embeddingsDb;
+      return;
+    }
+    const path = join("..", "..", "..", "app.asar.unpacked", "worker.js");
+    const embeddingWorkerPath = join("..", "..", "..", "app.asar.unpacked", "embeddings-worker.js");
 
-    this.worker = isDev
-      ? db
-      : await spawn<WorkerType<SQLDatabase>>(
-          new Worker(path, {
-            resourceLimits: { maxOldGenerationSizeMb: 32678, maxYoungGenerationSizeMb: 32678 },
-          }),
-        );
-    this.embeddingsWorker = isDev
-      ? embeddingsDb
-      : await spawn<WorkerType<EmbeddingsDatabase>>(
-          new Worker(embeddingWorkerPath, {
-            resourceLimits: { maxOldGenerationSizeMb: 32678, maxYoungGenerationSizeMb: 32678 },
-          }),
-        );
+    const opts: ThreadsWorkerOptions = {
+      resourceLimits: {
+        maxOldGenerationSizeMb: 65356,
+        maxYoungGenerationSizeMb: 65356,
+      },
+    };
+    this.worker = await spawn<WorkerType<SQLDatabase>>(new Worker(path, opts));
+    this.embeddingsWorker = await spawn<WorkerType<EmbeddingsDatabase>>(new Worker(embeddingWorkerPath, opts));
   };
 
   setupHandlers() {
@@ -56,6 +54,7 @@ class DbWorker {
     }
 
     handleIpc("loadVectorsIntoMemory", this.embeddingsWorker.loadVectorsIntoMemory);
+    handleIpc("embeddingsCacheSize", this.embeddingsWorker.embeddingsCacheSize);
   }
   isCopying = false;
   doesLocalDbCopyExist = async () => {