From b25f2ad51881198982c886d179d23669a9b4074a Mon Sep 17 00:00:00 2001 From: Alexander Alemayhu Date: Sat, 31 Aug 2024 17:15:22 +0200 Subject: [PATCH 1/7] Revert "Revert "chore: remove debug option" (#1590)" This reverts commit 322120f868fa439e0e37e2c250e4224e34d3ed22. --- src/lib/parser/DeckParser.ts | 40 +++++-- src/lib/parser/PrepareDeck.ts | 12 +-- src/test/test-utils.ts | 7 +- .../uploads/GeneratePackagesUseCase.ts | 101 ++---------------- src/usecases/uploads/getPackagesFromZip.ts | 68 ++++++++++++ src/usecases/uploads/worker.ts | 56 ++++++++++ 6 files changed, 176 insertions(+), 108 deletions(-) create mode 100644 src/usecases/uploads/getPackagesFromZip.ts create mode 100644 src/usecases/uploads/worker.ts diff --git a/src/lib/parser/DeckParser.ts b/src/lib/parser/DeckParser.ts index cc7fe66fc..6e96ca4b2 100644 --- a/src/lib/parser/DeckParser.ts +++ b/src/lib/parser/DeckParser.ts @@ -23,6 +23,14 @@ import { isFileNameEqual } from '../storage/types'; import { isImageFileEmbedable, isMarkdownFile } from '../storage/checks'; import { getFileContents } from './getFileContents'; import { handleNestedBulletPointsInMarkdown } from './handleNestedBulletPointsInMarkdown'; +import { checkFlashcardsLimits } from '../User/checkFlashcardsLimits'; + +export interface DeckParserInput { + name: string; + settings: Settings; + files: File[]; + noLimits: boolean; +} export class DeckParser { globalTags: cheerio.Cheerio | null; @@ -35,22 +43,27 @@ export class DeckParser { files: File[]; + noLimits: boolean; + public get name() { return this.payload[0].name; } - constructor(name: string, settings: Settings, files: File[]) { - this.settings = settings; - this.files = files || []; - this.firstDeckName = name; + constructor(input: DeckParserInput) { + this.settings = input.settings; + this.files = input.files || []; + this.firstDeckName = input.name; + this.noLimits = input.noLimits; this.globalTags = null; - const firstFile = this.files.find((file) => isFileNameEqual(file, name)); + const firstFile = this.files.find((file) => + isFileNameEqual(file, input.name) + ); - if (this.settings.nestedBulletPoints && isMarkdownFile(name)) { + if (this.settings.nestedBulletPoints && isMarkdownFile(input.name)) { const contents = getFileContents(firstFile, false); this.payload = handleNestedBulletPointsInMarkdown( - name, + input.name, contents?.toString(), this.settings.deckName, [], @@ -60,7 +73,7 @@ export class DeckParser { const contents = getFileContents(firstFile, true); this.payload = contents ? this.handleHTML( - name, + input.name, contents.toString(), this.settings.deckName || '', [] @@ -571,6 +584,8 @@ export class DeckParser { const parentUL = p; const parentClass = p.attr('class') || ''; + this.checkLimits(cards.length, []); + if (this.settings.toggleMode === 'open_toggle') { dom('details').attr('open', ''); } else if (this.settings.toggleMode === 'close_toggle') { @@ -651,10 +666,19 @@ export class DeckParser { lists.forEach((list) => { for (const child of dom(list).find('li')) { + this.checkLimits(cards.length, []); cards.push(new Note(dom(child).html() ?? '', '')); } }); return cards; } + + private checkLimits(cards: number, decks: Deck[]) { + checkFlashcardsLimits({ + cards: cards, + decks: decks, + paying: this.noLimits, + }); + } } diff --git a/src/lib/parser/PrepareDeck.ts b/src/lib/parser/PrepareDeck.ts index e91dcf7dd..72fea9fd2 100644 --- a/src/lib/parser/PrepareDeck.ts +++ b/src/lib/parser/PrepareDeck.ts @@ -1,7 +1,5 @@ -import { File } from '../anki/zip'; -import Settings from './Settings'; import getDeckFilename from '../anki/getDeckFilename'; -import { DeckParser } from './DeckParser'; +import { DeckParser, DeckParserInput } from './DeckParser'; import Deck from './Deck'; interface PrepareDeckResult { @@ -11,16 +9,14 @@ interface PrepareDeckResult { } export async function PrepareDeck( - fileName: string, - files: File[], - settings: Settings + input: DeckParserInput ): Promise { - const parser = new DeckParser(fileName, settings, files); + const parser = new DeckParser(input); if (parser.totalCardCount() === 0) { const apkg = await parser.tryExperimental(); return { - name: getDeckFilename(parser.name ?? fileName), + name: getDeckFilename(parser.name ?? input.name), apkg, deck: parser.payload, }; diff --git a/src/test/test-utils.ts b/src/test/test-utils.ts index adbcdd077..8c17e51aa 100644 --- a/src/test/test-utils.ts +++ b/src/test/test-utils.ts @@ -19,7 +19,12 @@ function loadFixture(fileName: string) { function configureParser(fileName: string, opts: Settings) { const info = loadFixture(fileName); - return new DeckParser(fileName, opts, info); + return new DeckParser({ + name: fileName, + settings: opts, + files: info, + noLimits: true, + }); } export async function getDeck(fileName: string, opts: Settings) { diff --git a/src/usecases/uploads/GeneratePackagesUseCase.ts b/src/usecases/uploads/GeneratePackagesUseCase.ts index f3808e014..6890ead8b 100644 --- a/src/usecases/uploads/GeneratePackagesUseCase.ts +++ b/src/usecases/uploads/GeneratePackagesUseCase.ts @@ -1,108 +1,27 @@ -import fs from 'fs'; - -import { ZipHandler } from '../../lib/anki/zip'; import Package from '../../lib/parser/Package'; import Settings from '../../lib/parser/Settings'; -import { - isCSVFile, - isHTMLFile, - isMarkdownFile, - isPlainText, - isZIPFile, -} from '../../lib/storage/checks'; import { UploadedFile } from '../../lib/storage/types'; - -import { Body } from 'aws-sdk/clients/s3'; -import { PrepareDeck } from '../../lib/parser/PrepareDeck'; -import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits'; +import { Worker } from 'worker_threads'; +import path from 'path'; export interface PackageResult { packages: Package[]; } -export const isFileSupported = (filename: string) => - isHTMLFile(filename) ?? - isMarkdownFile(filename) ?? - isPlainText(filename) ?? - isCSVFile(filename); - -const getPackagesFromZip = async ( - fileContents: Body | undefined, - paying: boolean, - settings: Settings -): Promise => { - const zipHandler = new ZipHandler(); - const packages = []; - - if (!fileContents) { - return { packages: [] }; - } - - zipHandler.build(fileContents as Uint8Array, paying); - - const fileNames = zipHandler.getFileNames(); - - let cardCount = 0; - for (const fileName of fileNames) { - if (isFileSupported(fileName)) { - const deck = await PrepareDeck(fileName, zipHandler.files, settings); - - if (deck) { - packages.push(new Package(deck.name, deck.apkg)); - cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0); - - // Checking the limit in place while iterating through the decks - checkFlashcardsLimits({ - cards: 0, - decks: deck.deck, - paying, - }); - } - } - - // Checking the limit in place while iterating through the files - checkFlashcardsLimits({ - cards: cardCount, - paying: paying, - }); - } - - return { packages }; -}; - class GeneratePackagesUseCase { - async execute( + execute( paying: boolean, files: UploadedFile[], settings: Settings ): Promise { - let packages: Package[] = []; + return new Promise((resolve, reject) => { + const data = { paying, files, settings }; + const workerPath = path.resolve(__dirname, './worker.js'); + const worker = new Worker(workerPath, { workerData: { data } }); - for (const file of files) { - const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer; - const filename = file.originalname; - const key = file.key; - - if (isFileSupported(filename)) { - const d = await PrepareDeck( - filename, - [{ name: filename, contents: fileContents }], - settings - ); - if (d) { - const pkg = new Package(d.name, d.apkg); - packages = packages.concat(pkg); - } - } else if (isZIPFile(filename) || isZIPFile(key)) { - const { packages: extraPackages } = await getPackagesFromZip( - fileContents, - paying, - settings - ); - packages = packages.concat(extraPackages); - } - } - return { packages }; + worker.on('message', (result: PackageResult) => resolve(result)); + worker.on('error', (error) => reject(error)); + }); } } diff --git a/src/usecases/uploads/getPackagesFromZip.ts b/src/usecases/uploads/getPackagesFromZip.ts new file mode 100644 index 000000000..dd25b7093 --- /dev/null +++ b/src/usecases/uploads/getPackagesFromZip.ts @@ -0,0 +1,68 @@ +import { Body } from 'aws-sdk/clients/s3'; +import Settings from '../../lib/parser/Settings'; +import { ZipHandler } from '../../lib/anki/zip'; +import { PrepareDeck } from '../../lib/parser/PrepareDeck'; +import Package from '../../lib/parser/Package'; +import { checkFlashcardsLimits } from '../../lib/User/checkFlashcardsLimits'; +import { PackageResult } from './GeneratePackagesUseCase'; +import { + isCSVFile, + isHTMLFile, + isMarkdownFile, + isPlainText, +} from '../../lib/storage/checks'; + +export const isFileSupported = (filename: string) => + isHTMLFile(filename) ?? + isMarkdownFile(filename) ?? + isPlainText(filename) ?? + isCSVFile(filename); + +export const getPackagesFromZip = async ( + fileContents: Body | undefined, + paying: boolean, + settings: Settings +): Promise => { + const zipHandler = new ZipHandler(); + const packages = []; + + if (!fileContents) { + return { packages: [] }; + } + + zipHandler.build(fileContents as Uint8Array, paying); + + const fileNames = zipHandler.getFileNames(); + + let cardCount = 0; + for (const fileName of fileNames) { + if (isFileSupported(fileName)) { + const deck = await PrepareDeck({ + name: fileName, + files: zipHandler.files, + settings, + noLimits: paying, + }); + + if (deck) { + packages.push(new Package(deck.name, deck.apkg)); + cardCount += deck.deck.reduce((acc, d) => acc + d.cards.length, 0); + + // Checking the limit in place while iterating through the decks + checkFlashcardsLimits({ + cards: 0, + decks: deck.deck, + paying, + }); + } + } + + // Checking the limit in place while iterating through the files + checkFlashcardsLimits({ + cards: cardCount, + paying: paying, + }); + } + + return { packages }; +}; diff --git a/src/usecases/uploads/worker.ts b/src/usecases/uploads/worker.ts new file mode 100644 index 000000000..290f3310a --- /dev/null +++ b/src/usecases/uploads/worker.ts @@ -0,0 +1,56 @@ +import { parentPort, workerData } from 'worker_threads'; +import { UploadedFile } from '../../lib/storage/types'; +import Settings from '../../lib/parser/Settings'; +import Package from '../../lib/parser/Package'; +import fs from 'fs'; +import { PrepareDeck } from '../../lib/parser/PrepareDeck'; +import { isZIPFile } from '../../lib/storage/checks'; +import { getPackagesFromZip, isFileSupported } from './getPackagesFromZip'; + +interface GenerationData { + paying: boolean; + files: UploadedFile[]; + settings: Settings; +} + +function doGenerationWork(data: GenerationData) { + console.log('doGenerationWork'); + return new Promise(async (resolve) => { + console.log('starting generation'); + const { paying, files, settings } = data; + let packages: Package[] = []; + + for (const file of files) { + const fileContents = file.path ? fs.readFileSync(file.path) : file.buffer; + const filename = file.originalname; + const key = file.key; + + if (isFileSupported(filename)) { + const d = await PrepareDeck({ + name: filename, + files: [{ name: filename, contents: fileContents }], + settings, + noLimits: paying, + }); + if (d) { + const pkg = new Package(d.name, d.apkg); + packages = packages.concat(pkg); + } + } else if (isZIPFile(filename) || isZIPFile(key)) { + const { packages: extraPackages } = await getPackagesFromZip( + fileContents, + paying, + settings + ); + packages = packages.concat(extraPackages); + } + } + resolve({ packages }); + }); +} + +doGenerationWork(workerData.data) + .then((result) => { + parentPort?.postMessage(result); + }) + .catch(parentPort?.postMessage); From db94d09cd7c2d536817ee9dbd9c1c41543c8348e Mon Sep 17 00:00:00 2001 From: Alexander Alemayhu Date: Sun, 1 Sep 2024 17:29:35 +0200 Subject: [PATCH 2/7] feat: wip --- .../SimpleUploadController.ts | 63 ------------------- .../SimpleUploadController/createPackages.ts | 16 ----- .../SimpleUploadController/createResponse.ts | 24 ------- src/routes/SimpleUploadRouter.ts | 17 ----- src/server.ts | 2 - src/services/UploadService.ts | 2 + 6 files changed, 2 insertions(+), 122 deletions(-) delete mode 100644 src/controllers/SimpleUploadController/SimpleUploadController.ts delete mode 100644 src/controllers/SimpleUploadController/createPackages.ts delete mode 100644 src/controllers/SimpleUploadController/createResponse.ts delete mode 100644 src/routes/SimpleUploadRouter.ts diff --git a/src/controllers/SimpleUploadController/SimpleUploadController.ts b/src/controllers/SimpleUploadController/SimpleUploadController.ts deleted file mode 100644 index 97ce5edc2..000000000 --- a/src/controllers/SimpleUploadController/SimpleUploadController.ts +++ /dev/null @@ -1,63 +0,0 @@ -import express from 'express'; - -import { sendError } from '../../lib/error/sendError'; -import { getLimitMessage } from '../../lib/misc/getLimitMessage'; -import { UploadedFile } from '../../lib/storage/types'; - -import { getUploadHandler } from '../../lib/misc/GetUploadHandler'; -import { createPackages } from './createPackages'; -import { CreatedDeck, createResponse } from './createResponse'; -import { isPaying } from '../../lib/isPaying'; - -const getPayingErrorMessage = () => { - return "There was an unknown error with your upload. Please try again. If the problem persists, please contact support@2anki.net."; -}; - -class SimpleUploadController { - async handleUpload(req: express.Request, res: express.Response) { - try { - const packages = await createPackages( - req.files as UploadedFile[], - isPaying(res.locals), - req.body - ); - const response: CreatedDeck[] = createResponse(packages); - return res.json(response); - } catch (err) { - if (err instanceof Error) { - return res.json({ - error: err.message, - }); - } - } - } - - file(req: express.Request, res: express.Response) { - try { - console.info('uploading file'); - const handleUploadEndpoint = getUploadHandler(res); - - handleUploadEndpoint(req, res, async (error) => { - if (error) { - let msg = error.message; - if (msg === 'File too large' && !isPaying(res.locals)) { - msg = getLimitMessage(); - } else if (isPaying(res.locals)) { - msg = getPayingErrorMessage(); - console.info('paying customer issue'); - sendError(error); - } else { - sendError(error); - } - return res.status(500).send(msg); - } - await this.handleUpload(req, res); - }); - } catch (error) { - sendError(error); - res.status(400); - } - } -} - -export default SimpleUploadController; diff --git a/src/controllers/SimpleUploadController/createPackages.ts b/src/controllers/SimpleUploadController/createPackages.ts deleted file mode 100644 index deaec3b14..000000000 --- a/src/controllers/SimpleUploadController/createPackages.ts +++ /dev/null @@ -1,16 +0,0 @@ -import Settings from '../../lib/parser/Settings'; -import GeneratePackagesUseCase from '../../usecases/uploads/GeneratePackagesUseCase'; -import { UploadedFile } from '../../lib/storage/types'; - -export const createPackages = async ( - files: UploadedFile[], - paying: boolean, - body: { [key: string]: string } = {} -) => { - const settings = new Settings(body); - - const useCase = new GeneratePackagesUseCase(); - const { packages } = await useCase.execute(paying, files, settings); - - return packages; -}; diff --git a/src/controllers/SimpleUploadController/createResponse.ts b/src/controllers/SimpleUploadController/createResponse.ts deleted file mode 100644 index 10e65d5d2..000000000 --- a/src/controllers/SimpleUploadController/createResponse.ts +++ /dev/null @@ -1,24 +0,0 @@ -import path from 'path'; -import fs from 'fs'; - -import Package from '../../lib/parser/Package'; -import Workspace from '../../lib/parser/WorkSpace'; - -export interface CreatedDeck { - name: string; - link: string; -} -export const createResponse = (packages: Package[]) => { - const workspace = new Workspace(true, 'fs'); - const basePath = `/download/${workspace.id}`; - const createdDecks = []; - for (const pkg of packages) { - const p = path.join(workspace.location, pkg.name); - fs.writeFileSync(p, pkg.apkg); - createdDecks.push({ - name: pkg.name, - link: `${basePath}/${pkg.name}`, - }); - } - return createdDecks; -}; diff --git a/src/routes/SimpleUploadRouter.ts b/src/routes/SimpleUploadRouter.ts deleted file mode 100644 index 779f2fe5d..000000000 --- a/src/routes/SimpleUploadRouter.ts +++ /dev/null @@ -1,17 +0,0 @@ -import express from 'express'; - -import SimpleUploadController from '../controllers/SimpleUploadController/SimpleUploadController'; -import RequireAllowedOrigin from './middleware/RequireAllowedOrigin'; - -const UploadRouter = () => { - const controller = new SimpleUploadController(); - const router = express.Router(); - - router.post('/api/simple-upload/file', RequireAllowedOrigin, (req, res) => - controller.file(req, res) - ); - - return router; -}; - -export default UploadRouter; diff --git a/src/server.ts b/src/server.ts index 96f70e2be..6fd9f0b6f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -27,7 +27,6 @@ import downloadRouter from './routes/DownloadRouter'; import favoriteRouter from './routes/FavoriteRouter'; import templatesRouter from './routes/TemplatesRouter'; import defaultRouter from './routes/DefaultRouter'; -import simpleUploadRouter from './routes/SimpleUploadRouter'; import webhookRouter from './routes/WebhookRouter'; import { sendError } from './lib/error/sendError'; @@ -75,7 +74,6 @@ const serve = async () => { app.use(downloadRouter()); app.use(favoriteRouter()); app.use(templatesRouter()); - app.use(simpleUploadRouter()); // Note: this has to be the last router app.use(defaultRouter()); diff --git a/src/services/UploadService.ts b/src/services/UploadService.ts index d6fcffcd8..b1ba8a8e1 100644 --- a/src/services/UploadService.ts +++ b/src/services/UploadService.ts @@ -43,6 +43,8 @@ class UploadService { settings ); + console.log('packages', packages); + const first = packages[0]; if (packages.length === 1) { if (!first.apkg) { From 84dc7a5daa39b04d796378b27e96354ff9d2c9b8 Mon Sep 17 00:00:00 2001 From: Alexander Alemayhu Date: Sun, 1 Sep 2024 20:10:18 +0200 Subject: [PATCH 3/7] fix: make the APKG generation thread safe Using the file system location is much more reliable than passing around buffer. --- src/controllers/DownloadController.ts | 7 ++++++- src/lib/parser/DeckParser.ts | 7 +++---- src/lib/parser/PrepareDeck.ts | 4 ++-- src/lib/parser/WorkSpace.ts | 13 ++++++++++++ src/pages/DownloadPage.tsx | 3 ++- src/services/UploadService.ts | 21 ++++++++----------- src/test/test-utils.ts | 4 +++- .../uploads/GeneratePackagesUseCase.ts | 6 ++++-- src/usecases/uploads/getPackagesFromZip.ts | 5 ++++- src/usecases/uploads/worker.ts | 8 +++++-- 10 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/controllers/DownloadController.ts b/src/controllers/DownloadController.ts index 38f3dbda1..a64007e84 100644 --- a/src/controllers/DownloadController.ts +++ b/src/controllers/DownloadController.ts @@ -61,7 +61,12 @@ class DownloadController { return; } - const page = DownloadPage({ id, files }); + console.log('files', files); + + const page = DownloadPage({ + id, + files: files.filter((file) => file.endsWith('.apkg')), + }); res.send(page); }); } else { diff --git a/src/lib/parser/DeckParser.ts b/src/lib/parser/DeckParser.ts index 6e96ca4b2..b9425cc3c 100644 --- a/src/lib/parser/DeckParser.ts +++ b/src/lib/parser/DeckParser.ts @@ -30,6 +30,7 @@ export interface DeckParserInput { settings: Settings; files: File[]; noLimits: boolean; + workspace: Workspace; } export class DeckParser { @@ -354,8 +355,7 @@ export class DeckParser { return card; } - build() { - const ws = new Workspace(true, 'fs'); + build(ws: Workspace) { const exporter = this.setupExporter(this.payload, ws.location); for (const d of this.payload) { @@ -475,9 +475,8 @@ export class DeckParser { return exporter.save(); } - tryExperimental() { + tryExperimental(ws: Workspace) { const fallback = new FallbackParser(this.files); - const ws = new Workspace(true, 'fs'); const exporter = this.setupExporter(this.payload, ws.location); this.payload = fallback.run(this.settings); diff --git a/src/lib/parser/PrepareDeck.ts b/src/lib/parser/PrepareDeck.ts index 72fea9fd2..65d57de2a 100644 --- a/src/lib/parser/PrepareDeck.ts +++ b/src/lib/parser/PrepareDeck.ts @@ -14,7 +14,7 @@ export async function PrepareDeck( const parser = new DeckParser(input); if (parser.totalCardCount() === 0) { - const apkg = await parser.tryExperimental(); + const apkg = await parser.tryExperimental(input.workspace); return { name: getDeckFilename(parser.name ?? input.name), apkg, @@ -22,7 +22,7 @@ export async function PrepareDeck( }; } - const apkg = await parser.build(); + const apkg = await parser.build(input.workspace); return { name: getDeckFilename(parser.name), apkg, diff --git a/src/lib/parser/WorkSpace.ts b/src/lib/parser/WorkSpace.ts index 7c5288544..768b3deae 100644 --- a/src/lib/parser/WorkSpace.ts +++ b/src/lib/parser/WorkSpace.ts @@ -22,6 +22,19 @@ class Workspace { fs.mkdirSync(this.location, { recursive: true }); } } + + public getFirstAPKG(): Promise { + return new Promise((resolve, reject) => { + fs.readdir(this.location, (err, files) => { + const apkg = files.find((file) => file.endsWith('.apkg')); + if (apkg) { + resolve(fs.readFileSync(path.join(this.location, apkg))); + } else { + reject(null); + } + }); + }); + } } export default Workspace; diff --git a/src/pages/DownloadPage.tsx b/src/pages/DownloadPage.tsx index de8c10763..2c18dcd68 100644 --- a/src/pages/DownloadPage.tsx +++ b/src/pages/DownloadPage.tsx @@ -44,6 +44,7 @@ export const DownloadPage = ({ id, files }: DownloadPageProps) => { }, downloadItemLink: {}, }; + const apkgFiles = files.filter((file) => file.endsWith('.apkg')); return ReactDOMServer.renderToStaticMarkup( @@ -56,7 +57,7 @@ export const DownloadPage = ({ id, files }: DownloadPageProps) => {