Vector DB

.yarn/versions/4bce8ecc.yml

@@ -20,5 +20,6 @@ undecided:
   - "@snickerdoodlelabs/synamint-extension-sdk"
   - "@snickerdoodlelabs/core-test-harness"
   - "@snickerdoodlelabs/utils"
+  - "@snickerdoodlelabs/vector-db"
   - "@snickerdoodlelabs/web-integration"
   - "@snickerdoodlelabs/web-integration-test"

packages/core/src/implementations/SnickerdoodleCore.module.ts

@@ -29,6 +29,7 @@ import {
   ICryptoUtilsType,
 } from "@snickerdoodlelabs/node-utils";
 import {
+  IIndexedDB,
   ITokenPriceRepository,
   ITokenPriceRepositoryType,
 } from "@snickerdoodlelabs/objects";
@@ -60,6 +61,10 @@ import {
   IPersistenceContextProviderType,
   NullCloudStorage,
   INullCloudStorageType,
+  DBContextProvider,
+  IIndexedDBContextProvider,
+  IIndexedDBContextProviderType,
+  IIndexedDBContext,
 } from "@snickerdoodlelabs/persistence";
 import {
   IQueryObjectFactory,
@@ -325,8 +330,8 @@ export const snickerdoodleCoreModule = new ContainerModule(
       .to(MonitoringService)
       .inSingletonScope();
     bind<IQuestionnaireService>(IQuestionnaireServiceType)
-    .to(QuestionnaireService)
-    .inSingletonScope();
+      .to(QuestionnaireService)
+      .inSingletonScope();
     bind<IDiscordService>(IDiscordServiceType)
       .to(DiscordService)
       .inSingletonScope();
@@ -462,6 +467,16 @@ export const snickerdoodleCoreModule = new ContainerModule(
       IPersistenceContextProviderType,
     ).toConstantValue(contextProvider);
 
+    const dbContextProvider = new DBContextProvider(
+      new LogUtils(),
+      new TimeUtils(),
+      new VolatileStorageSchemaProvider(configProvider),
+    );
+
+    bind<IIndexedDBContextProvider>(
+      IIndexedDBContextProviderType,
+    ).toConstantValue(dbContextProvider);
+
     bind<IBlockchainProvider>(IBlockchainProviderType)
       .to(BlockchainProvider)
       .inSingletonScope();

packages/core/src/implementations/SnickerdoodleCore.ts

@@ -92,9 +92,17 @@ import {
   URLString,
   INftMethods,
   IQuestionnaireMethods,
+  IVectorQuantizationMethods,
   NewQuestionnaireAnswer,
   JSONString,
   EExternalFieldKey,
+  ERecordKey,
+  IIndexedDB,
+  IIndexedDBType,
+  QuantizedTableId,
+  VectorRow,
+  VolatileStorageMetadata,
+  VersionedObject,
 } from "@snickerdoodlelabs/objects";
 import {
   IndexedDBVolatileStorage,
@@ -108,6 +116,11 @@ import {
   IStorageUtilsType,
   LocalStorageUtils,
 } from "@snickerdoodlelabs/utils";
+import {
+  VectorDB,
+  IVectorDB,
+  IVectorDBType,
+} from "@snickerdoodlelabs/vector-db";
 import { ethers } from "ethers";
 import { Container } from "inversify";
 import { ResultAsync } from "neverthrow";
@@ -186,6 +199,7 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
   public storage: IStorageMethods;
   public nft: INftMethods;
   public questionnaire: IQuestionnaireMethods;
+  public quantization: IVectorQuantizationMethods;
 
   public constructor(
     configOverrides?: IConfigOverrides,
@@ -220,6 +234,8 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
         .inSingletonScope();
     }
 
+    this.iocContainer.bind(IVectorDBType).to(VectorDB).inSingletonScope();
+
     // Setup the config
     if (configOverrides != null) {
       const configProvider =
@@ -718,6 +734,54 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
       },
     };
 
+    // Vector DB Methods --------------------------------------------------------------------
+    this.quantization = {
+      initialize: (template?: IIndexedDB) => {
+        const quantizationService =
+          this.iocContainer.get<IVectorDB>(IVectorDBType);
+
+        return quantizationService.initialize(template);
+      },
+
+      table: (name: string) => {
+        const quantizationService =
+          this.iocContainer.get<IVectorDB>(IVectorDBType);
+
+        return quantizationService.table(name);
+      },
+
+      quantizeTables: (
+        tableNames: ERecordKey[],
+        callbacks: ((
+          row: VolatileStorageMetadata<VersionedObject>,
+        ) => VectorRow)[],
+        outputName: QuantizedTableId,
+      ) => {
+        const quantizationService =
+          this.iocContainer.get<IVectorDB>(IVectorDBType);
+
+        return quantizationService.quantizeTable(
+          tableNames,
+          callbacks,
+          outputName,
+        );
+      },
+
+      viewTables: () => {
+        const quantizationService =
+          this.iocContainer.get<IVectorDB>(IVectorDBType);
+
+        return quantizationService.viewTables();
+      },
+
+      kmeans: (tableName: QuantizedTableId, k: number) => {
+        const quantizationService =
+          this.iocContainer.get<IVectorDB>(IVectorDBType);
+
+        return quantizationService.kmeans(tableName, k);
+      },
+    };
+
     // Questionnaire Methods --------------------------------------------------------------------
     this.questionnaire = {
       getAllQuestionnaires: (

packages/core/tsconfig.json

@@ -40,6 +40,9 @@
     },
     {
       "path": "../signatureVerification"
+    },
+    {
+      "path": "../vector-db"
     }
   ],
   "include": [

packages/docker/Dockerfile

@@ -28,6 +28,7 @@ COPY packages/static-web-integration/package.json /build/packages/static-web-int
 COPY packages/synamint-extension-sdk/package.json /build/packages/synamint-extension-sdk/package.json
 COPY packages/test-harness/package.json /build/packages/test-harness/package.json
 COPY packages/utils/package.json /build/packages/utils/package.json
+COPY packages/vector-db/package.json /build/packages/vector-db/package.json
 COPY packages/web-integration/package.json /build/packages/web-integration/package.json
 COPY packages/web-integration-test/package.json /build/packages/web-integration-test/package.json
 # RUN yarn set version berry

packages/mobile/metro.config.js

@@ -37,6 +37,9 @@ const extraNodeModules = {
   "@snickerdoodlelabs/signature-verification": path.resolve(
     path.join(__dirname, "../signatureVerification"),
   ),
+  "@snickerdoodlelabs/vector-db": path.resolve(
+    path.join(__dirname, "../vector-db"),
+  ),
   stream: require.resolve("stream-browserify"),
   crypto: require.resolve("react-native-quick-crypto"),
   zlib: require.resolve("browserify-zlib"),

packages/objects/src/businessObjects/KMeansResult.ts

@@ -0,0 +1,6 @@
+export interface KMeansResult {
+  clusters: number[];
+  centroids: number[][]; // { centroid: number[], error: number, size: number }[],
+  converged: boolean;
+  iterations: number;
+}

packages/objects/src/businessObjects/QuantizedTable.ts

@@ -0,0 +1,18 @@
+import { QuantizedTableId } from "@objects/primitives";
+
+export class QuantizedTable {
+  protected _table;
+  protected _id;
+  public constructor(name: QuantizedTableId, data: number[][]) {
+    this._table = data;
+    this._id = name;
+  }
+
+  id(): QuantizedTableId {
+    return this._id;
+  }
+
+  table(): number[][] {
+    return this._table;
+  }
+}

packages/objects/src/businessObjects/index.ts

@@ -22,6 +22,7 @@ export * from "@objects/businessObjects/IndexerSupportSummary.js";
 export * from "@objects/businessObjects/Insight.js";
 export * from "@objects/businessObjects/InsightWithProof.js";
 export * from "@objects/businessObjects/Invitation.js";
+export * from "@objects/businessObjects/KMeansResult.js";
 export * from "@objects/businessObjects/MarketplaceListing.js";
 export * from "@objects/businessObjects/MetatransactionSignatureRequest.js";
 export * from "@objects/businessObjects/NftHoldings.js";
@@ -33,6 +34,7 @@ export * from "@objects/businessObjects/PageInvitation.js";
 export * from "@objects/businessObjects/PagingRequest.js";
 export * from "@objects/businessObjects/ParsedBackupFileName.js";
 export * from "@objects/businessObjects/PortfolioUpdate.js";
+export * from "@objects/businessObjects/QuantizedTable.js";
 export * from "@objects/businessObjects/Questionnaire.js";
 export * from "@objects/businessObjects/RequestForData.js";
 export * from "@objects/businessObjects/QueryDeliveryItems.js";

packages/objects/src/errors/VectorDBError.ts

@@ -0,0 +1,9 @@
+import { BaseError } from "@objects/errors/BaseError.js";
+import errorCodes from "@objects/errors/errorCodes.js";
+
+export class VectorDBError extends BaseError {
+  protected errorCode: string = errorCodes[VectorDBError.name];
+  constructor(message: string, public src?: unknown) {
+    super(message, 500, errorCodes[VectorDBError.name], src, false);
+  }
+}

packages/objects/src/errors/index.ts

@@ -32,6 +32,7 @@ export * from "@objects/errors/TwitterError.js";
 export * from "@objects/errors/UnauthorizedError.js";
 export * from "@objects/errors/UninitializedError.js";
 export * from "@objects/errors/UnsupportedLanguageError.js";
+export * from "@objects/errors/VectorDBError.js";
 
 export * from "@objects/errors/blockchain/index.js";
 export * from "@objects/errors/sdqlErrors/index.js";

packages/objects/src/interfaces/IIndexedDB.ts

@@ -0,0 +1,118 @@
+import { ResultAsync } from "neverthrow";
+
+import {
+  VersionedObject,
+  VolatileStorageMetadata,
+} from "@objects/businessObjects";
+import { PersistenceError } from "@objects/errors";
+import { IVolatileCursor } from "@objects/interfaces";
+import { VolatileStorageKey } from "@objects/primitives";
+
+export interface IIndexedDB {
+  initialize(): ResultAsync<IDBDatabase, PersistenceError>;
+  close(): ResultAsync<void, PersistenceError>;
+  persist(): ResultAsync<boolean, PersistenceError>;
+  clear(): ResultAsync<void, PersistenceError>;
+  clearObjectStore(name: string): ResultAsync<void, PersistenceError>;
+  putObject<T>(name: string, obj: T): ResultAsync<void, PersistenceError>;
+  removeObject<T extends VersionedObject>(
+    name: string,
+    key: string,
+  ): ResultAsync<VolatileStorageMetadata<T> | null, PersistenceError>;
+
+  getObject<T extends VersionedObject>(
+    name: string,
+    key: VolatileStorageKey,
+    _includeDeleted?: boolean,
+  ): ResultAsync<VolatileStorageMetadata<T> | null, PersistenceError>;
+
+  getCursor<T extends VersionedObject>(
+    name: string,
+    index?: VolatileStorageKey,
+    query?: string | number,
+    direction?: IDBCursorDirection | undefined,
+    mode?: IDBTransactionMode,
+  ): ResultAsync<IVolatileCursor<T>, PersistenceError>;
+
+  deleteDatabase(database: string): ResultAsync<void, PersistenceError>;
+
+  getAll<T extends VersionedObject>(
+    storeName: string,
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getAllByIndex<T extends VersionedObject>(
+    name: string,
+    index: VolatileStorageKey,
+    query: IDBValidKey | IDBKeyRange,
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getAllKeys<T>(
+    name: string,
+    index?: VolatileStorageKey,
+    query?: IDBValidKey | IDBKeyRange,
+    count?: number | undefined,
+  ): ResultAsync<T[], PersistenceError>;
+
+  get<T extends VersionedObject>(
+    name: string,
+    {
+      index,
+      query,
+      count,
+      id,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      count?: number;
+      id?: IDBValidKey;
+    },
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getKeys(
+    name: string,
+    {
+      index,
+      query,
+      count,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      count?: number;
+    },
+  ): ResultAsync<IDBValidKey[], PersistenceError>;
+
+  getCursor2<T extends VersionedObject>(
+    name: string,
+    {
+      index,
+      query,
+      lowerCount,
+      upperCount,
+      latest,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      lowerCount?: number;
+      upperCount?: number;
+      latest?: boolean;
+    },
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  countRecords(
+    name: string,
+    {
+      index,
+      query,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | undefined;
+    },
+  ): ResultAsync<number, PersistenceError>;
+
+  getKey<T extends VersionedObject>(
+    tableName: string,
+    obj: VolatileStorageMetadata<T>,
+  ): ResultAsync<VolatileStorageKey | null, PersistenceError>;
+}
+
+export const IIndexedDBType = Symbol.for("IIndexedDB");