feat: Implement worker threads for upload processing

package.json

@@ -47,7 +47,6 @@
     "get-notion-object-title": "^0.2.0",
     "html-to-text": "^9.0.4",
     "jsonwebtoken": "^9.0.0",
-    "jszip": "^3.10.1",
     "knex": "^3.1.0",
     "metascraper": "^5.34.7",
     "metascraper-description": "^5.39.0",

src/controllers/DownloadController.ts

@@ -61,7 +61,10 @@ class DownloadController {
           return;
         }
 
-        const page = DownloadPage({ id, files });
+        const page = DownloadPage({
+          id,
+          files: files.filter((file) => file.endsWith('.apkg')),
+        });
         res.send(page);
       });
     } else {

src/lib/anki/getDeckFilename.test.ts

@@ -10,7 +10,5 @@ test("does not append .apkg extension if it's already there", () => {
 });
 
 test("uses package name if it's available", () => {
-  expect(getDeckFilename(new Package('foo', Buffer.alloc(0)))).toEqual(
-    'foo.apkg'
-  );
+  expect(getDeckFilename(new Package('foo'))).toEqual('foo.apkg');
 });

src/lib/anki/zip.tsx

@@ -1,11 +1,8 @@
-import JSZip from 'jszip';
 import { strFromU8, unzipSync } from 'fflate';
-import Package from '../parser/Package';
 import { Body } from 'aws-sdk/clients/s3';
 import { renderToStaticMarkup } from 'react-dom/server';
 import { getUploadLimits } from '../misc/getUploadLimits';
 import { isHTMLFile, isMarkdownFile } from '../storage/checks';
-import getDeckFilename from './getDeckFilename';
 
 interface File {
   name: string;
@@ -60,17 +57,6 @@ class ZipHandler {
   getFileNames() {
     return this.fileNames;
   }
-
-  static toZip(decks: Package[], advertisment: string | null) {
-    const zip = new JSZip();
-    for (const d of decks) {
-      zip.file(getDeckFilename(d), d.apkg);
-    }
-    if (advertisment) {
-      zip.file('README.html', advertisment);
-    }
-    return zip.generateAsync({ type: 'nodebuffer' });
-  }
 }
 
 export { ZipHandler, File };

src/lib/parser/DeckParser.ts

@@ -23,6 +23,15 @@ import { isFileNameEqual } from '../storage/types';
 import { isImageFileEmbedable, isMarkdownFile } from '../storage/checks';
 import { getFileContents } from './getFileContents';
 import { handleNestedBulletPointsInMarkdown } from './handleNestedBulletPointsInMarkdown';
+import { checkFlashcardsLimits } from '../User/checkFlashcardsLimits';
+
+export interface DeckParserInput {
+  name: string;
+  settings: Settings;
+  files: File[];
+  noLimits: boolean;
+  workspace: Workspace;
+}
 
 export class DeckParser {
   globalTags: cheerio.Cheerio | null;
@@ -35,22 +44,27 @@ export class DeckParser {
 
   files: File[];
 
+  noLimits: boolean;
+
   public get name() {
     return this.payload[0].name;
   }
 
-  constructor(name: string, settings: Settings, files: File[]) {
-    this.settings = settings;
-    this.files = files || [];
-    this.firstDeckName = name;
+  constructor(input: DeckParserInput) {
+    this.settings = input.settings;
+    this.files = input.files || [];
+    this.firstDeckName = input.name;
+    this.noLimits = input.noLimits;
     this.globalTags = null;
 
-    const firstFile = this.files.find((file) => isFileNameEqual(file, name));
+    const firstFile = this.files.find((file) =>
+      isFileNameEqual(file, input.name)
+    );
 
-    if (this.settings.nestedBulletPoints && isMarkdownFile(name)) {
+    if (this.settings.nestedBulletPoints && isMarkdownFile(input.name)) {
       const contents = getFileContents(firstFile, false);
       this.payload = handleNestedBulletPointsInMarkdown(
-        name,
+        input.name,
         contents?.toString(),
         this.settings.deckName,
         [],
@@ -60,7 +74,7 @@ export class DeckParser {
       const contents = getFileContents(firstFile, true);
       this.payload = contents
         ? this.handleHTML(
-            name,
+            input.name,
             contents.toString(),
             this.settings.deckName || '',
             []
@@ -341,8 +355,7 @@ export class DeckParser {
     return card;
   }
 
-  build() {
-    const ws = new Workspace(true, 'fs');
+  build(ws: Workspace) {
     const exporter = this.setupExporter(this.payload, ws.location);
 
     for (const d of this.payload) {
@@ -462,9 +475,8 @@ export class DeckParser {
     return exporter.save();
   }
 
-  tryExperimental() {
+  tryExperimental(ws: Workspace) {
     const fallback = new FallbackParser(this.files);
-    const ws = new Workspace(true, 'fs');
     const exporter = this.setupExporter(this.payload, ws.location);
 
     this.payload = fallback.run(this.settings);
@@ -571,6 +583,8 @@ export class DeckParser {
       const parentUL = p;
       const parentClass = p.attr('class') || '';
 
+      this.checkLimits(cards.length, []);
+
       if (this.settings.toggleMode === 'open_toggle') {
         dom('details').attr('open', '');
       } else if (this.settings.toggleMode === 'close_toggle') {
@@ -651,10 +665,19 @@ export class DeckParser {
 
     lists.forEach((list) => {
       for (const child of dom(list).find('li')) {
+        this.checkLimits(cards.length, []);
         cards.push(new Note(dom(child).html() ?? '', ''));
       }
     });
 
     return cards;
   }
+
+  private checkLimits(cards: number, decks: Deck[]) {
+    checkFlashcardsLimits({
+      cards: cards,
+      decks: decks,
+      paying: this.noLimits,
+    });
+  }
 }

src/lib/parser/Package.ts

@@ -1,11 +1,8 @@
 class Package {
   name: string;
 
-  apkg: Buffer;
-
-  constructor(name: string, apkg: Buffer) {
+  constructor(name: string) {
     this.name = name;
-    this.apkg = apkg;
   }
 }