upload feature

pages are uploaded now to a cloud storage
(s3 or openstack swift with s3-like api enabled)

Dont use s3, its not even cheap, go and get your services at your local cloud company or any other not-that-big provider.
Also you can always write me if you want to hear my opinion. Peace.

.env.example

@@ -9,4 +9,14 @@ TYPEORM_PORT = 3306
 TYPEORM_USERNAME = m2k_user
 TYPEORM_PASSWORD = m2k_password
 TYPEORM_DATABASE = manga2kindle
-# more database related config can be edited directly in the src/config/typeorm/default.config.ts file
+# more database related config can be edited directly in the src/config/typeorm/default.config.ts file
+
+# S3 STORAGE (or compatible)
+S3_ACCESS_KEY_ID = 1a2b3c4d5e6f7g8h9i
+S3_SECRET_ACCESS_KEY = 1a2b3c4d5e6f7g8h9i
+S3_BUCKET = manga2kindle_image_bucket
+S3_ENDPOINT = https://s3.mycloudprovider.example
+S3_REGION = eu-example
+# Only compatible with OpenStack Swift, in seconds
+# if you are using AWS-S3 remember to set a lifecycle to avoid old files eat space if a worker dont delete them
+S3_TIME_TO_LIVE = 3600

package.json

@@ -42,6 +42,7 @@
     "@tsed/swagger": "6.1.5",
     "@tsed/typeorm": "6.1.5",
     "ajv": "6.12.6",
+    "aws-sdk": "^2.799.0",
     "body-parser": "1.19.0",
     "compression": "1.7.4",
     "cookie-parser": "1.4.5",
@@ -51,6 +52,7 @@
     "express": "4.17.1",
     "mariadb": "2.5.1",
     "method-override": "3.0.0",
+    "multer": "^1.4.2",
     "mysql": "^2.14.1",
     "reflect-metadata": "^0.1.10",
     "typeorm": "0.2.28"
@@ -64,6 +66,7 @@
     "@types/dotenv": "^8.2.0",
     "@types/express": "4.17.8",
     "@types/method-override": "0.0.31",
+    "@types/multer": "^1.4.4",
     "@types/node": "14.14.6",
     "@typescript-eslint/eslint-plugin": "4.6.0",
     "@typescript-eslint/parser": "4.6.0",

src/Server.ts

@@ -10,6 +10,7 @@ import "@tsed/ajv";
 import "@tsed/swagger";
 import "@tsed/typeorm";
 import typeormConfig from "./config/typeorm";
+import MulterS3Storage from "./modules/S3StorageEngine";
 
 export const rootDir = __dirname;
 
@@ -28,7 +29,10 @@ export const rootDir = __dirname;
     }
   ],
   typeorm: typeormConfig,
-  exclude: ["**/*.spec.ts"]
+  exclude: ["**/*.spec.ts"],
+  multer: {
+    storage: new MulterS3Storage()
+  }
 })
 export class Server {
   @Inject()

src/config/s3/index.ts

@@ -0,0 +1,9 @@
+import { Credentials, Endpoint, S3 } from "aws-sdk";
+
+const s3 = new S3();
+s3.config.credentials = new Credentials(process.env.S3_ACCESS_KEY_ID as string, process.env.S3_SECRET_ACCESS_KEY as string);
+s3.endpoint = new Endpoint(process.env.S3_ENDPOINT as string);
+s3.config.endpoint = process.env.S3_ENDPOINT;
+s3.config.region = process.env.S3_REGION;
+
+export default s3;

src/controllers/UploadController.ts

@@ -0,0 +1,54 @@
+import { Controller, MultipartFile, PathParams, PlatformMulterFile, Post } from "@tsed/common";
+import { NotFound, BadRequest, InternalServerError, Exception } from "@tsed/exceptions";
+import { Description, Returns, Summary } from "@tsed/schema";
+import { isNaturalNumber } from "../modules/DataValidation";
+import S3Storage from "../modules/S3Storage";
+import { StatusService } from "../services/StatusService";
+
+@Controller("/upload")
+export class UploadController {
+  constructor(private statusService: StatusService) {}
+
+  @Post("/:id/:page")
+  @Summary("Add new chapter page")
+  @Description(
+    'Upload chapter pages, one by one.<br>This call has an extra parameter named "file" in the body, it is a multipart to attach a page.'
+  )
+  @(Returns(201).Description("Created, no response expected"))
+  @Returns(400, BadRequest)
+  @Returns(404, NotFound)
+  async post(
+    @Description("A status (chapter) ID")
+    @PathParams("id")
+    id: number,
+    @Description("A page number")
+    @PathParams("page")
+    page: number,
+    @Description("A page file")
+    @MultipartFile("file")
+    file: PlatformMulterFile
+  ): Promise<string | Exception> {
+    return new Promise<string | Exception>(async (resolve, reject) => {
+      try {
+        if (!isNaturalNumber(id)) {
+          throw new BadRequest("ID is not a Number");
+        }
+        if (!isNaturalNumber(page)) {
+          throw new BadRequest("Page is not a Number");
+        }
+
+        if (!(await this.statusService.exists(id))) {
+          throw new NotFound("Chapter ID was not found");
+        }
+
+        resolve();
+      } catch (error) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        new S3Storage().deleteFile(file.path, (error: any, metadata?: any) => {
+          if (error) reject(new InternalServerError("An error ocurred while managing the file (" + metadata + ")"));
+          else reject(error);
+        });
+      }
+    });
+  }
+}

src/modules/S3Storage.ts

@@ -0,0 +1,60 @@
+import { PlatformMulterFile } from "@tsed/common";
+import { Exception } from "@tsed/exceptions";
+import { AWSError, Request as S3Request, S3 } from "aws-sdk";
+import s3 from "../config/s3";
+
+export default class S3Storage {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  public putFile(path: string, file: Buffer, cb: (error: any, metadata?: any) => void): void {
+    const params: S3.Types.PutObjectRequest = {
+      Bucket: process.env.S3_BUCKET as string,
+      Key: path,
+      Body: file
+    };
+
+    if (file.byteLength > 10485760) {
+      // 10MB
+      return cb(new Exception(400, "File too big"));
+    }
+
+    s3.putObject(params)
+      .on("build", (req: S3Request<S3.PutObjectOutput, AWSError>) => {
+        req.httpRequest.headers["X-Delete-After"] = process.env.S3_TIME_TO_LIVE || "3600";
+      })
+      .send((err: AWSError, data: S3.PutObjectOutput) => {
+        if (err) return cb(err);
+        else cb(null, data);
+      });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  public deleteFile(path: string, cb: (error: any, metadata?: any) => void): void {
+    const params: S3.Types.PutObjectRequest = {
+      Bucket: process.env.S3_BUCKET as string,
+      Key: path
+    };
+
+    s3.deleteObject(params, (err: AWSError, data: S3.DeleteObjectOutput) => {
+      if (err) return cb(err);
+      else cb(null, data);
+    });
+  }
+
+  public MulterFileToBuffer(file: PlatformMulterFile): Promise<Buffer> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const bufs: any[] = [];
+
+    return new Promise<Buffer>((resolve, reject) => {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      file.stream.on("data", function (chunk: any) {
+        bufs.push(chunk);
+      });
+      file.stream.on("end", function () {
+        resolve(Buffer.concat(bufs));
+      });
+      file.stream.on("error", function (err: Error) {
+        reject(err);
+      });
+    });
+  }
+}

src/modules/S3StorageEngine.ts

@@ -0,0 +1,35 @@
+import { PlatformMulterFile, Req } from "@tsed/common";
+import { StorageEngine } from "multer";
+import S3Storage from "./S3Storage";
+
+export default class MulterS3Storage implements StorageEngine {
+  private s3Utils = new S3Storage();
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  _handleFile = (req: Req, file: PlatformMulterFile, cb: (error: any, metadata?: any) => void): void => {
+    file.path = req.params.id + "/" + req.params.page + this.getFileExtension(file.originalname);
+
+    this.s3Utils.MulterFileToBuffer(file).then((bufferFile: Buffer) => {
+      this.s3Utils.putFile(file.path, bufferFile, cb);
+    });
+  };
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  _removeFile = (req: Req, file: PlatformMulterFile, cb: (error: any, metadata?: any) => void): void => {
+    this.s3Utils.deleteFile(file.path, cb);
+  };
+
+  private getFileExtension(filename: string): string | undefined {
+    let extension = filename.split(".").pop();
+    if (extension) {
+      extension = "." + extension;
+    }
+    return extension;
+  }
+}
+
+export function storageEngine(): MulterS3Storage {
+  return new MulterS3Storage();
+}
+
+export type ContentTypeFunction = (req: Request, file: Express.Multer.File) => string | undefined;

src/services/StatusService.ts

@@ -28,4 +28,16 @@ export class StatusService implements AfterRoutesInit {
     }
     return this.repository.findById(id);
   }
+
+  async exists(id: number): Promise<boolean> {
+    if (!isNaturalNumber(id)) {
+      throw new Error("Non Natural Number");
+    }
+
+    const element = await this.repository.findById(id);
+
+    if (element) return true;
+
+    return false;
+  }
 }