diff --git a/.env.example b/.env.example index 2aa0592..ede54d1 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,14 @@ TYPEORM_PORT = 3306 TYPEORM_USERNAME = m2k_user TYPEORM_PASSWORD = m2k_password TYPEORM_DATABASE = manga2kindle -# more database related config can be edited directly in the src/config/typeorm/default.config.ts file \ No newline at end of file +# more database related config can be edited directly in the src/config/typeorm/default.config.ts file + +# S3 STORAGE (or compatible) +S3_ACCESS_KEY_ID = 1a2b3c4d5e6f7g8h9i +S3_SECRET_ACCESS_KEY = 1a2b3c4d5e6f7g8h9i +S3_BUCKET = manga2kindle_image_bucket +S3_ENDPOINT = https://s3.mycloudprovider.example +S3_REGION = eu-example +# Only compatible with OpenStack Swift, in seconds +# if you are using AWS-S3 remember to set a lifecycle to avoid old files eat space if a worker dont delete them +S3_TIME_TO_LIVE = 3600 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 60ac21e..90066c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -433,6 +433,15 @@ "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==", "dev": true }, + "@types/multer": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.4.tgz", + "integrity": "sha512-wdfkiKBBEMTODNbuF3J+qDDSqJxt50yB9pgDiTcFew7f97Gcc7/sM4HR66ofGgpJPOALWOqKAch4gPyqEXSkeQ==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, "@types/node": { "version": "14.14.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.6.tgz", @@ -751,6 +760,68 @@ "lodash": "^4.17.14" } }, + "aws-sdk": { + "version": "2.799.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.799.0.tgz", + "integrity": "sha512-NYAoiNU+bJXhlJsC0rFqrmD5t5ho7/VxldmziP6HLPYHfOCI9Uvk6UVjfPmhLWPm0mHnIxhsHqmsNGyjhHNYmw==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -2055,6 +2126,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "execa": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz", @@ -2789,6 +2865,11 @@ } } }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -3785,6 +3866,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -4737,6 +4823,22 @@ "punycode": "^2.1.0" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "url-parse-lax": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", diff --git a/package.json b/package.json index f487cb3..ca8a9e8 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@tsed/swagger": "6.1.5", "@tsed/typeorm": "6.1.5", "ajv": "6.12.6", + "aws-sdk": "^2.799.0", "body-parser": "1.19.0", "compression": "1.7.4", "cookie-parser": "1.4.5", @@ -51,6 +52,7 @@ "express": "4.17.1", "mariadb": "2.5.1", "method-override": "3.0.0", + "multer": "^1.4.2", "mysql": "^2.14.1", "reflect-metadata": "^0.1.10", "typeorm": "0.2.28" @@ -64,6 +66,7 @@ "@types/dotenv": "^8.2.0", "@types/express": "4.17.8", "@types/method-override": "0.0.31", + "@types/multer": "^1.4.4", "@types/node": "14.14.6", "@typescript-eslint/eslint-plugin": "4.6.0", "@typescript-eslint/parser": "4.6.0", diff --git a/src/Server.ts b/src/Server.ts index ffa00cc..d73f177 100644 --- a/src/Server.ts +++ b/src/Server.ts @@ -10,6 +10,7 @@ import "@tsed/ajv"; import "@tsed/swagger"; import "@tsed/typeorm"; import typeormConfig from "./config/typeorm"; +import MulterS3Storage from "./modules/S3StorageEngine"; export const rootDir = __dirname; @@ -28,7 +29,10 @@ export const rootDir = __dirname; } ], typeorm: typeormConfig, - exclude: ["**/*.spec.ts"] + exclude: ["**/*.spec.ts"], + multer: { + storage: new MulterS3Storage() + } }) export class Server { @Inject() diff --git a/src/config/s3/index.ts b/src/config/s3/index.ts new file mode 100644 index 0000000..d7e4590 --- /dev/null +++ b/src/config/s3/index.ts @@ -0,0 +1,9 @@ +import { Credentials, Endpoint, S3 } from "aws-sdk"; + +const s3 = new S3(); +s3.config.credentials = new Credentials(process.env.S3_ACCESS_KEY_ID as string, process.env.S3_SECRET_ACCESS_KEY as string); +s3.endpoint = new Endpoint(process.env.S3_ENDPOINT as string); +s3.config.endpoint = process.env.S3_ENDPOINT; +s3.config.region = process.env.S3_REGION; + +export default s3; diff --git a/src/controllers/UploadController.ts b/src/controllers/UploadController.ts new file mode 100644 index 0000000..9502e67 --- /dev/null +++ b/src/controllers/UploadController.ts @@ -0,0 +1,54 @@ +import { Controller, MultipartFile, PathParams, PlatformMulterFile, Post } from "@tsed/common"; +import { NotFound, BadRequest, InternalServerError, Exception } from "@tsed/exceptions"; +import { Description, Returns, Summary } from "@tsed/schema"; +import { isNaturalNumber } from "../modules/DataValidation"; +import S3Storage from "../modules/S3Storage"; +import { StatusService } from "../services/StatusService"; + +@Controller("/upload") +export class UploadController { + constructor(private statusService: StatusService) {} + + @Post("/:id/:page") + @Summary("Add new chapter page") + @Description( + 'Upload chapter pages, one by one.
This call has an extra parameter named "file" in the body, it is a multipart to attach a page.' + ) + @(Returns(201).Description("Created, no response expected")) + @Returns(400, BadRequest) + @Returns(404, NotFound) + async post( + @Description("A status (chapter) ID") + @PathParams("id") + id: number, + @Description("A page number") + @PathParams("page") + page: number, + @Description("A page file") + @MultipartFile("file") + file: PlatformMulterFile + ): Promise { + return new Promise(async (resolve, reject) => { + try { + if (!isNaturalNumber(id)) { + throw new BadRequest("ID is not a Number"); + } + if (!isNaturalNumber(page)) { + throw new BadRequest("Page is not a Number"); + } + + if (!(await this.statusService.exists(id))) { + throw new NotFound("Chapter ID was not found"); + } + + resolve(); + } catch (error) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new S3Storage().deleteFile(file.path, (error: any, metadata?: any) => { + if (error) reject(new InternalServerError("An error ocurred while managing the file (" + metadata + ")")); + else reject(error); + }); + } + }); + } +} diff --git a/src/modules/S3Storage.ts b/src/modules/S3Storage.ts new file mode 100644 index 0000000..e7e491e --- /dev/null +++ b/src/modules/S3Storage.ts @@ -0,0 +1,60 @@ +import { PlatformMulterFile } from "@tsed/common"; +import { Exception } from "@tsed/exceptions"; +import { AWSError, Request as S3Request, S3 } from "aws-sdk"; +import s3 from "../config/s3"; + +export default class S3Storage { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public putFile(path: string, file: Buffer, cb: (error: any, metadata?: any) => void): void { + const params: S3.Types.PutObjectRequest = { + Bucket: process.env.S3_BUCKET as string, + Key: path, + Body: file + }; + + if (file.byteLength > 10485760) { + // 10MB + return cb(new Exception(400, "File too big")); + } + + s3.putObject(params) + .on("build", (req: S3Request) => { + req.httpRequest.headers["X-Delete-After"] = process.env.S3_TIME_TO_LIVE || "3600"; + }) + .send((err: AWSError, data: S3.PutObjectOutput) => { + if (err) return cb(err); + else cb(null, data); + }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public deleteFile(path: string, cb: (error: any, metadata?: any) => void): void { + const params: S3.Types.PutObjectRequest = { + Bucket: process.env.S3_BUCKET as string, + Key: path + }; + + s3.deleteObject(params, (err: AWSError, data: S3.DeleteObjectOutput) => { + if (err) return cb(err); + else cb(null, data); + }); + } + + public MulterFileToBuffer(file: PlatformMulterFile): Promise { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const bufs: any[] = []; + + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.stream.on("data", function (chunk: any) { + bufs.push(chunk); + }); + file.stream.on("end", function () { + resolve(Buffer.concat(bufs)); + }); + file.stream.on("error", function (err: Error) { + reject(err); + }); + }); + } +} diff --git a/src/modules/S3StorageEngine.ts b/src/modules/S3StorageEngine.ts new file mode 100644 index 0000000..acd62ad --- /dev/null +++ b/src/modules/S3StorageEngine.ts @@ -0,0 +1,35 @@ +import { PlatformMulterFile, Req } from "@tsed/common"; +import { StorageEngine } from "multer"; +import S3Storage from "./S3Storage"; + +export default class MulterS3Storage implements StorageEngine { + private s3Utils = new S3Storage(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _handleFile = (req: Req, file: PlatformMulterFile, cb: (error: any, metadata?: any) => void): void => { + file.path = req.params.id + "/" + req.params.page + this.getFileExtension(file.originalname); + + this.s3Utils.MulterFileToBuffer(file).then((bufferFile: Buffer) => { + this.s3Utils.putFile(file.path, bufferFile, cb); + }); + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _removeFile = (req: Req, file: PlatformMulterFile, cb: (error: any, metadata?: any) => void): void => { + this.s3Utils.deleteFile(file.path, cb); + }; + + private getFileExtension(filename: string): string | undefined { + let extension = filename.split(".").pop(); + if (extension) { + extension = "." + extension; + } + return extension; + } +} + +export function storageEngine(): MulterS3Storage { + return new MulterS3Storage(); +} + +export type ContentTypeFunction = (req: Request, file: Express.Multer.File) => string | undefined; diff --git a/src/services/StatusService.ts b/src/services/StatusService.ts index 16eaf46..18cbc3f 100644 --- a/src/services/StatusService.ts +++ b/src/services/StatusService.ts @@ -28,4 +28,16 @@ export class StatusService implements AfterRoutesInit { } return this.repository.findById(id); } + + async exists(id: number): Promise { + if (!isNaturalNumber(id)) { + throw new Error("Non Natural Number"); + } + + const element = await this.repository.findById(id); + + if (element) return true; + + return false; + } }