Merge pull request #51 from dcSpark/feature/incognito

incognito tools

tools/download-page/metadata.json

@@ -12,7 +12,12 @@
     ],
     "configurations": {
       "type": "object",
-      "properties": {},
+      "properties": {
+        "chromePath": {
+          "type": "string",
+          "description": "The path to the Chrome executable"
+        }
+      },
       "required": []
     },
     "parameters": {

tools/download-page/tool.ts

@@ -1,25 +1,68 @@
+
+import chromePaths from 'npm:[email protected]';
 import TurndownService from 'npm:[email protected]';
-import axios from 'npm:[email protected]';
+import { addExtra } from 'npm:[email protected]';
+import rebrowserPuppeteer from 'npm:[email protected]';
+import StealthPlugin from 'npm:[email protected]';
+
+import { getHomePath } from './shinkai-local-support.ts';
+
+type Configurations = {
+  chromePath?: string;
+};
 
-type Configurations = {};
 type Parameters = {
   url: string;
+  incognito?: boolean;
 };
 
 type Result = { markdown: string };
+
 export type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;
 
+const puppeteer = addExtra(rebrowserPuppeteer as any);
+const pluginStealth = StealthPlugin();
+pluginStealth.enabledEvasions.delete('chrome.loadTimes');
+pluginStealth.enabledEvasions.delete('chrome.runtime');
+puppeteer.use(pluginStealth);
+
 export const run: Run<Configurations, Parameters, Result> = async (
-  _configurations: Configurations,
+  configurations: Configurations,
   parameters: Parameters,
 ): Promise<Result> => {
-  try {
-    const response = await axios.get(parameters.url);
-    const turndownService = new TurndownService();
-    const markdown = turndownService.turndown(response.data);
-    return Promise.resolve({ markdown });
-  } catch (error) {
-    console.log('error', error);
-    return Promise.resolve({ markdown: '' });
+  const chromePath =
+    configurations?.chromePath ||
+    Deno.env.get('CHROME_PATH') ||
+    chromePaths.chrome ||
+    chromePaths.chromium;
+  if (!chromePath) {
+    throw new Error('Chrome path not found');
   }
+  console.log({ chromePath })
+  const browser = await puppeteer.launch({
+    executablePath: chromePath,
+    args: ['--disable-blink-features=AutomationControlled'],
+  });
+
+  const page = await browser.newPage();
+
+  console.log("Navigating to website...");
+  await page.goto(parameters.url);
+
+  console.log('Waiting for the page to load...');
+  await page.waitForNetworkIdle();
+
+  console.log('Extracting HTML content...');
+  const html = await page.content();
+
+  console.log('Closing browser...');
+  await browser.close();
+
+  console.log('Saving HTML to file...');
+  Deno.writeTextFileSync(await getHomePath() + '/download-page.html', html);
+
+  console.log('Converting HTML to Markdown...');
+  const turndownService = new TurndownService();
+  const markdown = turndownService.turndown(html);
+  return Promise.resolve({ markdown });
 };

tools/perplexity/metadata.json

@@ -6,7 +6,7 @@
     "version": "1.0.0",
     "keywords": [
       "perplexity",
-      "shinkai"
+      "search"
     ],
     "configurations": {
       "type": "object",

tools/perplexity/tool.ts

@@ -1,7 +1,9 @@
-import * as playwright from 'npm:[email protected]';
+
 import chromePaths from 'npm:[email protected]';
 import TurndownService from 'npm:[email protected]';
-import { defineConfig } from 'npm:[email protected]/test';
+import { addExtra } from 'npm:[email protected]';
+import rebrowserPuppeteer from 'npm:[email protected]';
+import StealthPlugin from 'npm:[email protected]';
 
 type Configurations = {
   chromePath?: string;
@@ -10,52 +12,49 @@ type Parameters = {
   query: string;
 };
 type Result = { response: string };
+
 export type Run<C extends Record<string, any>, I extends Record<string, any>, R extends Record<string, any>> = (config: C, inputs: I) => Promise<R>;
 
+const puppeteer = addExtra(rebrowserPuppeteer as any);
+const pluginStealth = StealthPlugin();
+
+pluginStealth.enabledEvasions.delete('chrome.loadTimes');
+pluginStealth.enabledEvasions.delete('chrome.runtime');
+
+puppeteer.use(pluginStealth);
+
 export const run: Run<Configurations, Parameters, Result> = async (
   configurations,
   params,
 ): Promise<Result> => {
-  defineConfig({
-    use: {
-      actionTimeout: 60 * 1000,
-      navigationTimeout: 60 * 1000,
-    },
-  });
   const chromePath =
     configurations?.chromePath ||
     Deno.env.get('CHROME_PATH') ||
     chromePaths.chrome ||
     chromePaths.chromium;
-  const browser = await playwright['chromium'].launch({
+  if (!chromePath) {
+    throw new Error('Chrome path not found');
+  }
+  const browser = await puppeteer.launch({
     executablePath: chromePath,
+    args: ['--disable-blink-features=AutomationControlled'],
   });
-  const context = await browser.newContext({
-    viewport: { width: 1280, height: 800 }, // Set viewport size
-    userAgent:
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36', // Set Mac user agent
-  });
-  const page = await context.newPage();
+  const page = await browser.newPage();
 
   console.log("Navigating to Perplexity's website...");
   await page.goto('https://www.perplexity.ai/');
 
   console.log('Waiting for the page to load...');
-  await page.waitForTimeout(2500);
+  await page.waitForNetworkIdle({ timeout: 2500 });
 
   console.log('Filling textarea with query:', params.query);
-  await page.fill('textarea', params.query);
+  await page.type('textarea', params.query);
 
   try {
     console.log('trying to click app popup');
-    await page.click('button:has(svg[data-icon="xmark"])', { timeout: 2000 });
+    await page.click('button:has(svg[data-icon="xmark"])');
   } catch (_) {
     console.log('unable to find the x button to close the popup');
-    /*
-      We do nothing, so we have two cases:
-      - the code continue and fails later because we are not able to click the "submit" button
-      - the code continue and it just works because the app was changed and the popup doesn't exists
-    */
   }
 
   console.log('Clicking the button with the specified SVG...');
@@ -65,7 +64,7 @@ export const run: Run<Configurations, Parameters, Result> = async (
   await page.waitForSelector('button:has(svg[data-icon="arrow-right"])');
 
   console.log('Waiting for results to load...');
-  await page.waitForSelector('div:has-text("Related")');
+  await page.waitForSelector('text=Related');
 
   console.log('Extracting HTML content...');
   const htmlContent = await page.evaluate(() => {