Feat/engt 1717

.eslintrc

@@ -2,7 +2,12 @@
   "parser": "@typescript-eslint/parser", // Specifies the ESLint parser
   "parserOptions": {
     "ecmaVersion": 2020, // Allows for the parsing of modern ECMAScript features
-    "sourceType": "module" // Allows for the use of imports
+    "sourceType": "module", // Allows for the use of imports
+    // "babelOptions": {
+    //   "plugins": [
+    //     "@babel/plugin-syntax-import-assertions"
+    //   ],
+    // },
   },
   "extends": [
     "plugin:import/errors",

documentation/ai-scraper/README.md

@@ -78,7 +78,7 @@ participant SJR as Scraper Job Repository
 participant LLM as LLM Scraper
 participant PU as PromptUtils
 participant LLMProvider as LLM Provider
-participant T as Text Preprocessor
+participant Pre as HTML Preprocessor
 participant DTU as DomainTaskUtils
 
 loop Periodically
@@ -95,8 +95,8 @@ loop Periodically
             PU-->LLM: Prompt
 
             loop for each job
-                LLM->>T: convert HTML to text and minimize
-                T-->>LLM: minimzed Text
+                LLM->>Pre: convert HTML to text and minimize
+                Pre-->>LLM: minimzed Text
                 LLM->>PU: add to Prompt if have token budget
             end
 
@@ -118,3 +118,66 @@ end
 
 1. Web-workers
 2. IPFS to collect URLs?
+
+### Prompt Builder
+
+```mermaid
+classDiagram
+
+    class PromptDirector {
+        +makePurchaseHistoryPrompt(data) Prompt
+    }
+
+    PromptDirector --> PromptBuilder
+
+    class PromptBuilder {
+        <<interface>>
+        +setExemplars(exemplars)
+        +setRole(role)
+        +setQuestion(question)
+        +setAnswerStructure(structure)
+        +setData(data)
+        +getPrompt() Prompt
+    }
+
+    class PurchaseHistoryPromptBuilder {
+
+    }
+    PromptBuilder <|-- PurchaseHistoryPromptBuilder
+
+    class ShoppingCartPromptBuilder {
+
+    }
+    PromptBuilder <|-- ShoppingCartPromptBuilder
+
+    %% Collection prompts
+
+    class CollectionPromptBuilder {
+
+        <<interface>>
+    }
+    PromptBuilder <|-- CollectionPromptBuilder
+
+    class ProductCollectionPromptBuilder {
+
+    }
+    CollectionPromptBuilder <|-- ProductCollectionPromptBuilder
+
+    class GameCollectionPromptBuilder {
+
+    }
+    CollectionPromptBuilder <|-- GameCollectionPromptBuilder
+
+
+    %% Single Item Prompts
+    class ItemDetailsPromptBuilder {
+
+    }
+    PromptBuilder <|-- ItemDetailsPromptBuilder
+
+    class OrderDetailsPromptBuilder {
+
+    }
+    ItemDetailsPromptBuilder <|-- OrderDetailsPromptBuilder
+
+```

documentation/persistence layer/README.md

@@ -44,13 +44,18 @@ We add the migrator definition to the same file where we defined the Animal clas
         }
     }
 ```
-3. Every entity requires a schema which is analogous to table definitions in SQL. We add the schema to [VolatileStorageSchema.ts](./../../packages/persistence/src/volatile/VolatileStorageSchema.ts) by adding an object of type [VolatileTableIndex](./../../packages/persistence/src/volatile/VolatileTableIndex.ts). 
+3. Every entity requires a schema which is analogous to table definitions in SQL. We add the schema to [VolatileStorageSchemaProvider.ts](./../../packages/persistence/src/volatile/VolatileStorageSchemaProvider.ts) by adding an object of type [VolatileTableIndex](./../../packages/persistence/src/volatile/VolatileTableIndex.ts). 
 ```
   new VolatileTableIndex(
     ERecordKey.ANIMAL, // The name of our object store / table
     "id", // primary key field.
     false, // false disables the auto-increment key generator. 
     new AnimalMigrator(), // migrator that our database client will use to convert data into animal objects.
+
+    EBackupPriority.NORMAL, // Backup priority
+    3600 * 1000, // Backup Interval in milliseconds
+    config.backupChunkSizeTarget,
+    [], // Index
   ),
 
 ```
@@ -65,7 +70,12 @@ We add the migrator definition to the same file where we defined the Animal clas
     "id", // primary key field.
     false, // false disables the auto-increment key generator. 
     new AnimalMigrator(),
-    [['name', false], ['someOtherField', false], [['comp1', 'comp2'], true]
+
+    EBackupPriority.NORMAL, // Backup priority
+    3600 * 1000, // Backup Interval in milliseconds
+    config.backupChunkSizeTarget,
+
+    [['name', false], ['someOtherField', false], [['comp1', 'comp2'], true]]
   ),
 
 
@@ -78,25 +88,20 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A
 **Add an animal**: We add a new object to the store by wrapping it in a [VolatileStorageMetadata](./../../packages/objects/src/businessObjects/VolatileStorageMetadata.ts) object.
 ```
     const myDog = new Animal("XX12", "Tom");
-    const metadata = new VolatileStorageMetadata<Animal>(
-        EBackupPriority.NORMAL,
-        myDog,
-        Animal.CURRENT_VERSION,
-    );
-    return this.persistence.updateRecord(ERecordKey.ANIMAL, metadata);
+    return this.persistence.updateRecord(ERecordKey.ANIMAL, myDog);
 ```
 
 **Update an animal**: Update works exactly the same way as adding a new object. The engine will update and existing object if an object with the same primary key exists.
 
 **Delete an animal**: Current we only support deleting by the primary key. The value of the primary key needs to be wrapped in a [VolatileStorageKey](./../../packages/objects/src/primitives/VolatileStorageKey.ts) object.
 
 ```
-    return this.persistence.deleteRecord(ERecordKey.ANIMAL, VolatileStorageKey("XX12"), EBackupPriority.NORMAL); // Deletes Tom from the animal store.
+    return this.persistence.deleteRecord(ERecordKey.ANIMAL, "XX12", EBackupPriority.NORMAL); // Deletes Tom from the animal store.
 ```
 
 **Find an animal by primary key**:
 ```
-    return this.persistence.getObject(ERecordKey.ANIMAL, VolatileStorageKey("XX12"), EBackupPriority.NORMAL); // Deletes Tom from the animal store.
+    return this.persistence.getObject(ERecordKey.ANIMAL, "XX12", EBackupPriority.NORMAL); // Deletes Tom from the animal store.
 ```
 
 **Get all animals**:
@@ -105,7 +110,7 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A
 ```
 **Get all animals by an index**:
 ```
-    return this.persistence.getAllByIndex(ERecordKey.ANIMAL, "name", IDBValidKey("Tom")); // Analogous to the getCursor function. But it returns all the objects.
+    return this.persistence.getAllByIndex(ERecordKey.ANIMAL, "name", "Tom"); // Analogous to the getCursor function. But it returns all the objects.
 ```
 **Get all primary keys**:
 ```
@@ -115,7 +120,7 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A
 **Get cursor**:
 Cursors can return all the objects or a subset matching an index field. For details, please check [IndexDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB) document.
 ```
-    return this.persistence.getCursor(ERecordKey.ANIMAL, "name", IDBValidKey("Tom")); // will return a cursor with all the Toms. 
+    return this.persistence.getCursor(ERecordKey.ANIMAL, "name", "Tom"); // will return a cursor with all the Toms. 
 ```
 
 

documentation/shopping-data/README.md

@@ -0,0 +1 @@
+# TODO

packages/ai-scraper/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@snickerdoodlelabs/ai-scraper",
   "version": "0.0.17",
-  "description": "Utilities for parsing and understanding SDQL Queries",
+  "description": "Web scraper for Data Wallet browser extension",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -41,9 +41,14 @@
   "dependencies": {
     "@snickerdoodlelabs/common-utils": "workspace:^",
     "@snickerdoodlelabs/objects": "workspace:^",
+    "@snickerdoodlelabs/persistence": "workspace:^",
+    "@snickerdoodlelabs/shopping-data": "workspace:^",
     "ethers": "^5.6.6",
+    "html-to-text": "^9.0.5",
     "inversify": "^6.0.1",
+    "js-tiktoken": "^1.0.7",
     "neverthrow": "^5.1.0",
-    "neverthrow-result-utils": "^2.0.2"
+    "neverthrow-result-utils": "^2.0.2",
+    "openai": "^4.0.1"
   }
 }

packages/ai-scraper/src/implementations/business/ChatGPTProvider.ts

@@ -0,0 +1,101 @@
+import { ILogUtilsType, ILogUtils } from "@snickerdoodlelabs/common-utils";
+import { LLMError, LLMResponse, Prompt } from "@snickerdoodlelabs/objects";
+import { inject, injectable } from "inversify";
+import {
+  getEncoding,
+  encodingForModel,
+  TiktokenModel,
+  Tiktoken,
+} from "js-tiktoken";
+import { ResultAsync, okAsync, errAsync } from "neverthrow";
+import OpenAI from "openai";
+import {
+  ChatCompletion,
+  ChatCompletionCreateParamsNonStreaming,
+  CompletionCreateParamsNonStreaming,
+} from "openai/resources/chat";
+
+import {
+  ILLMProvider,
+  IOpenAIUtils,
+  IOpenAIUtilsType,
+  IScraperConfigProvider,
+  IScraperConfigProviderType,
+} from "@ai-scraper/interfaces/index.js";
+
+@injectable()
+export class ChatGPTProvider implements ILLMProvider {
+  private chatModel: TiktokenModel = "gpt-3.5-turbo";
+  private temperature: number;
+  private chatEncoder: Tiktoken;
+  // private timeout = 5 * 60 * 1000; // 5 minutes
+
+  public constructor(
+    @inject(IScraperConfigProviderType)
+    private configProvider: IScraperConfigProvider,
+    @inject(ILogUtilsType)
+    private logUtils: ILogUtils,
+    @inject(IOpenAIUtilsType)
+    private openAIUtils: IOpenAIUtils,
+  ) {
+    this.temperature = 0.1;
+    this.chatEncoder = encodingForModel(this.chatModel);
+  }
+
+  public defaultMaxTokens(): number {
+    return 4096;
+  }
+
+  public maxTokens(model: TiktokenModel): number {
+    switch (model) {
+      case "gpt-3.5-turbo":
+        return 4096;
+      case "gpt-4":
+        return 8192;
+    }
+    return 0;
+  }
+
+  public getPromptTokens(prompt: Prompt): ResultAsync<number, Error> {
+    const tokens = this.chatEncoder.encode(prompt); // This might take a while
+    return okAsync(tokens.length);
+  }
+
+  public executePrompt(prompt: Prompt): ResultAsync<LLMResponse, LLMError> {
+    const messages = [
+      { role: "system", content: "You are an helpful assistant." },
+      { role: "user", content: prompt },
+    ];
+
+    return this.chatOnce(messages);
+  }
+
+  private getClient(): ResultAsync<OpenAI, LLMError> {
+    return this.configProvider.getConfig().andThen((config) => {
+      try {
+        const clientOptions = {
+          apiKey: config.scraper.OPENAI_API_KEY,
+          timeout: config.scraper.timeout,
+        };
+        // this.logUtils.debug("ChatGPTProvider", "constructor", clientOptions);
+        return okAsync(new OpenAI(clientOptions));
+      } catch (e) {
+        return errAsync(new LLMError((e as Error).message, e));
+      }
+    });
+  }
+
+  private chatOnce(messages): ResultAsync<LLMResponse, LLMError> {
+    return this.getClient().andThen((openai) => {
+      const params: ChatCompletionCreateParamsNonStreaming = {
+        model: this.chatModel,
+        messages: messages,
+        temperature: this.temperature,
+      };
+      const completionResult =
+        this.openAIUtils.createChatCompletionNonStreaming(openai, params);
+
+      return this.openAIUtils.parseCompletionResult(completionResult);
+    });
+  }
+}

packages/ai-scraper/src/implementations/business/HTMLPreProcessor.ts

@@ -0,0 +1,75 @@
+import {
+  ELanguageCode,
+  HTMLString,
+  ScraperError,
+} from "@snickerdoodlelabs/objects";
+import { compile, convert } from "html-to-text";
+import { injectable } from "inversify";
+import { ResultAsync, okAsync } from "neverthrow";
+
+import { IHTMLPreProcessor } from "@ai-scraper/interfaces/index.js";
+
+type Converter = (html: string) => string;
+
+@injectable()
+export class HTMLPreProcessor implements IHTMLPreProcessor {
+  private converter: Converter;
+  private converterWithImages: Converter;
+  private converterWithLinks: Converter;
+  private headConverter: Converter;
+
+  public constructor() {
+    const options = {
+      selectors: [
+        { selector: "a", options: { ignoreHref: true } },
+        { selector: "img", format: "skip" },
+      ],
+    };
+    this.converter = compile(options);
+
+    const optionsImages = {
+      baseElements: { selectors: ["body"] },
+      selectors: [{ selector: "a", options: { ignoreHref: true } }],
+    };
+
+    this.converterWithImages = compile(optionsImages);
+
+    const optionsLinks = {
+      baseElements: { selectors: ["body"] },
+      selectors: [{ selector: "img", format: "skip" }],
+    };
+    this.converterWithLinks = compile(optionsLinks);
+
+    const headOptions = {
+      baseElements: { selectors: ["head"] },
+    };
+    this.headConverter = compile(headOptions);
+  }
+
+  public getLanguage(
+    html: HTMLString,
+  ): ResultAsync<ELanguageCode, ScraperError> {
+    return okAsync(ELanguageCode.English); // TODO parse html tag for language. if not found, use third party library (nlp.js) to detect language such as google translate.
+  }
+
+  public htmlToText(
+    html: HTMLString,
+    options: unknown | null,
+  ): ResultAsync<string, ScraperError> {
+    if (options == null) {
+      return okAsync(this.converter(html));
+    } else {
+      return okAsync(convert(html, options));
+    }
+  }
+  public htmlToTextWithImages(
+    html: HTMLString,
+  ): ResultAsync<string, ScraperError> {
+    return okAsync(this.converterWithImages(html));
+  }
+  public htmlToTextWithLinks(
+    html: HTMLString,
+  ): ResultAsync<string, ScraperError> {
+    return okAsync(this.converterWithLinks(html));
+  }
+}