feat: experimental pdf support

We are using the [Google Vertex AI API](https://cloud.google.com/vertex-ai/docs/reference/rest).

package.json

@@ -31,6 +31,7 @@
   "license": "MIT",
   "dependencies": {
     "@2anki/csv-to-apkg": "^1.4.4",
+    "@google-cloud/vertexai": "^1.9.0",
     "@notionhq/client": "^2.2.13",
     "@sendgrid/mail": "^8.1.3",
     "aws-sdk": "^2.1502.0",

src/controllers/SettingsController/supportedOptions.ts

@@ -118,6 +118,12 @@ const supportedOptions = (): CardOption[] => {
       'Enable conversion of bullet and sub bullet points in Markdown. If you are a Obsidian user, enable this',
       false
     ),
+    new CardOption(
+      'vertex-ai-pdf-questions',
+      'Generate Questions from PDFs',
+      'Use Vertex AI API to generate questions from PDFs. This is a paid feature and if enabled will send your notes to Google Cloud.',
+      false
+    ),
   ];
 
   return v.filter(Boolean);

src/lib/anki/zip.tsx

@@ -2,7 +2,7 @@ import { strFromU8, unzipSync } from 'fflate';
 import { Body } from 'aws-sdk/clients/s3';
 import { renderToStaticMarkup } from 'react-dom/server';
 import { getUploadLimits } from '../misc/getUploadLimits';
-import { isHTMLFile, isMarkdownFile } from '../storage/checks';
+import { isHTMLFile, isMarkdownFile, isPDFFile } from '../storage/checks';
 
 interface File {
   name: string;
@@ -46,6 +46,14 @@ class ZipHandler {
     for (const name of this.fileNames) {
       const file = loadedZip[name];
       let contents = file;
+
+      /**
+       * For now disable batch processing of PDF files. We only want single uploads to avoid creating too many requests.
+       */
+      if (name.includes('__MACOSX/') || isPDFFile(name)) {
+        continue;
+      }
+
       if ((isHTMLFile(name) || isMarkdownFile(name)) && contents) {
         this.files.push({ name, contents: strFromU8(file) });
       } else if (contents) {

src/lib/parser/PrepareDeck.ts

@@ -1,6 +1,8 @@
 import getDeckFilename from '../anki/getDeckFilename';
 import { DeckParser, DeckParserInput } from './DeckParser';
 import Deck from './Deck';
+import { isPDFFile } from '../storage/checks';
+import { convertPDFToHTML } from './experimental/VertexAPI/convertPDFToHTML';
 
 interface PrepareDeckResult {
   name: string;
@@ -11,6 +13,17 @@ interface PrepareDeckResult {
 export async function PrepareDeck(
   input: DeckParserInput
 ): Promise<PrepareDeckResult> {
+  if (input.noLimits) {
+    // Check for PDF files and convert their contents to HTML
+    for (const file of input.files) {
+      if (isPDFFile(file.name) && file.contents) {
+        file.contents = await convertPDFToHTML(
+          file.contents.toString('base64')
+        );
+      }
+    }
+  }
+
   const parser = new DeckParser(input);
 
   if (parser.totalCardCount() === 0) {

src/lib/parser/Settings/Settings.ts

@@ -66,6 +66,8 @@ export class Settings {
 
   readonly nestedBulletPoints: boolean;
 
+  readonly vertexAIPDFQuestions: boolean;
+
   constructor(input: { [key: string]: string }) {
     this.deckName = input.deckName;
     if (this.deckName && !this.deckName.trim()) {
@@ -97,6 +99,7 @@ export class Settings {
     this.parentBlockId = input.parentBlockId;
     this.pageEmoji = input['page-emoji'] || 'first_emoji';
     this.addNotionLink = input['add-notion-link'] === 'true';
+    this.vertexAIPDFQuestions = input['vertex-ai-pdf-questions'] === 'true';
     /* Is this really needed? */
     if (this.parentBlockId) {
       this.addNotionLink = true;

src/lib/parser/experimental/VertexAPI/convertPDFToHTML.ts

@@ -0,0 +1,90 @@
+import path from 'path';
+import fs from 'fs';
+
+import {
+  GenerateContentRequest,
+  HarmBlockThreshold,
+  HarmCategory,
+  VertexAI,
+} from '@google-cloud/vertexai';
+
+export const convertPDFToHTML = async (pdf: string): Promise<string> => {
+  const vertexAI = new VertexAI({
+    project: 'notion-to-anki',
+    location: 'europe-west3',
+  });
+  const model = 'gemini-1.5-flash-002';
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+    generationConfig: {
+      maxOutputTokens: 8192,
+      temperature: 1,
+      topP: 0.95,
+    },
+    safetySettings: [
+      {
+        category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+      },
+    ],
+  });
+
+  const document1 = {
+    inlineData: {
+      mimeType: 'application/pdf',
+      data: pdf,
+    },
+  };
+
+  const text1 = {
+    text: fs
+      .readFileSync(
+        path.join(
+          __dirname,
+          '../../../../../../pdf-to-html-api',
+          'instructions.txt'
+        )
+      )
+      .toString(),
+  };
+
+  const req: GenerateContentRequest = {
+    contents: [{ role: 'user', parts: [document1, text1] }],
+  };
+
+  let htmlContent = '';
+  try {
+    const streamingResp = await generativeModel.generateContentStream(req);
+    for await (const item of streamingResp.stream) {
+      if (
+        item.candidates &&
+        item.candidates[0].content &&
+        item.candidates[0].content.parts
+      ) {
+        htmlContent += item.candidates[0].content.parts
+          .map((part) => part.text)
+          .join('');
+      }
+    }
+  } catch (error) {
+    console.error('Error generating content stream:', error);
+
+    // const workSpace = process.cwd();
+    // const outputPath = path.join(workSpace, 'output.html');
+    // fs.writeFileSync(outputPath, htmlContent);
+    // console.log(outputPath);
+  }
+  return htmlContent;
+};