Skip to content

Commit

Permalink
refactor uploader plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@f0x52
f0x52 committed May 20, 2024
1 parent 5c1ab79 commit 4b0ad89
Show file tree
Hide file tree
Showing 4 changed files with 246 additions and 192 deletions.
5 changes: 4 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
"./public/**"
],
"dependencies": {
"@fastify/busboy": "1.0.0",
"bcryptjs": "2.4.3",
"chalk": "4.1.2",
"cheerio": "1.0.0-rc.12",
Expand All @@ -71,10 +70,13 @@
"linkify-it": "3.0.3",
"lodash": "4.17.21",
"mime-types": "2.1.34",
"multer": "1.4.5-lts.1",
"nanoid": "5.0.7",
"node-forge": "1.3.0",
"package-json": "7.0.0",
"read": "1.0.7",
"read-chunk": "3.2.0",
"resolve-path": "1.4.0",
"semver": "7.5.2",
"socket.io": "4.6.1",
"tlds": "1.228.0",
Expand Down Expand Up @@ -106,6 +108,7 @@
"@types/mime-types": "2.1.1",
"@types/mocha": "9.1.1",
"@types/mousetrap": "1.6.15",
"@types/multer": "1.4.11",
"@types/node": "17.0.45",
"@types/read": "0.0.32",
"@types/semver": "7.3.9",
Expand Down
264 changes: 95 additions & 169 deletions server/plugins/uploader.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
import Config from "../config";
import busboy, {BusboyHeaders} from "@fastify/busboy";
import {v4 as uuidv4} from "uuid";
import path from "path";
import fs from "fs";
import fs from "fs/promises";
import fileType from "file-type";
import readChunk from "read-chunk";
import crypto from "crypto";
import isUtf8 from "is-utf8";
import log from "../log";
import contentDisposition from "content-disposition";
import type {Socket} from "socket.io";
import {Request, Response} from "express";
import type { Socket } from "socket.io";
import { Request, Response, NextFunction } from "express";
import Client from "../client";
import resolvePath from "resolve-path";
import multer from "multer";
import { nanoid } from "nanoid";

declare module 'express' {
interface Request {
_username?: string
}
}

// Map of allowed mime types to their respecive default filenames
// that will be rendered in browser without forcing them to be downloaded
Expand All @@ -35,35 +42,42 @@ const inlineContentDispositionTypes = {
"video/webm": "video.webm",
};

const uploadTokens = new Map();
interface UploadToken {
name: string,
timeout: NodeJS.Timeout,
};

const uploadTokens = new Map<string, UploadToken>();

class Uploader {
constructor(socket: Socket) {
constructor(client: Client, socket: Socket) {
socket.on("upload:auth", () => {
const token = uuidv4();
const token = nanoid();

socket.emit("upload:auth", token);

// Invalidate the token in one minute
const timeout = Uploader.createTokenTimeout(token);

uploadTokens.set(token, timeout);
uploadTokens.set(token, {
name: client.name,
timeout,
});
});

socket.on("upload:ping", (token) => {
if (typeof token !== "string") {
return;
}

let timeout = uploadTokens.get(token);
const storedToken = uploadTokens.get(token);

if (!timeout) {
if (!storedToken) {
return;
}

clearTimeout(timeout);
timeout = Uploader.createTokenTimeout(token);
uploadTokens.set(token, timeout);
clearTimeout(storedToken.timeout);
storedToken.timeout = Uploader.createTokenTimeout(token);
});
}

Expand All @@ -73,24 +87,31 @@ class Uploader {

// TODO: type
static router(this: void, express: any) {
express.get("/uploads/:name/:slug*?", Uploader.routeGetFile);
express.post("/uploads/new/:token", Uploader.routeUploadFile);
express.get("/uploads/:nick/:mediaId/:slug*?", Uploader.routeGetFile);
express.post("/uploads/new/:token",
Uploader.consumeToken,
Uploader.uploadMiddleware,
Uploader.afterUpload
);
}

static async routeGetFile(this: void, req: Request, res: Response) {
const name = req.params.name;
const { nick, mediaId } = req.params;

const nameRegex = /^[0-9a-f]{16}$/;
const uploadPath = Config.getFileUploadPath();

const unsafePath = path.join(nick, mediaId);

let filePath, detectedMimeType;

if (!nameRegex.test(name)) {
try {
filePath = resolvePath(uploadPath, unsafePath);
detectedMimeType = await Uploader.getFileType(filePath);
} catch (err: any) {
log.error("uploaded file access error: %s", err.message);
return res.status(404).send("Not found");
}

const folder = name.substring(0, 2);
const uploadPath = Config.getFileUploadPath();
const filePath = path.join(uploadPath, folder, name);
let detectedMimeType = await Uploader.getFileType(filePath);

// doesn't exist
if (detectedMimeType === null) {
return res.status(404).send("Not found");
Expand Down Expand Up @@ -125,167 +146,72 @@ class Uploader {
}

res.setHeader("Content-Disposition", disposition);
res.setHeader("Cache-Control", "max-age=86400");
res.contentType(detectedMimeType);

return res.sendFile(filePath);
return res.sendFile(filePath, {
root: uploadPath,
maxAge: 86400
});
}

static routeUploadFile(this: void, req: Request, res: Response) {
let busboyInstance: NodeJS.WritableStream | busboy | null | undefined;
let uploadUrl: string | URL;
let randomName: string;
let destDir: fs.PathLike;
let destPath: fs.PathLike | null;
let streamWriter: fs.WriteStream | null;

const doneCallback = () => {
// detach the stream and drain any remaining data
if (busboyInstance) {
req.unpipe(busboyInstance);
req.on("readable", req.read.bind(req));

busboyInstance.removeAllListeners();
busboyInstance = null;
}

// close the output file stream
if (streamWriter) {
streamWriter.end();
streamWriter = null;
}
};

const abortWithError = (err: any) => {
doneCallback();

// if we ended up erroring out, delete the output file from disk
if (destPath && fs.existsSync(destPath)) {
fs.unlinkSync(destPath);
destPath = null;
}

return res.status(400).json({error: err.message});
};

static consumeToken(this: void, req: Request, res: Response, next: NextFunction) {
// if the authentication token is incorrect, bail out
if (uploadTokens.delete(req.params.token) !== true) {
return abortWithError(Error("Invalid upload token"));
}
const storedToken = uploadTokens.get(req.params.token);

// if the request does not contain any body data, bail out
if (req.headers["content-length"] && parseInt(req.headers["content-length"]) < 1) {
return abortWithError(Error("Length Required"));
if (storedToken === undefined) {
return res.status(400).json({ error: "Invalid upload token" });
}

// Only allow multipart, as busboy can throw an error on unsupported types
if (
!(
req.headers["content-type"] &&
req.headers["content-type"].startsWith("multipart/form-data")
)
) {
return abortWithError(Error("Unsupported Content Type"));
}
uploadTokens.delete(req.params.token);
req._username = storedToken.name;

// create a new busboy processor, it is wrapped in try/catch
// because it can throw on malformed headers
try {
busboyInstance = new busboy({
headers: req.headers as BusboyHeaders,
limits: {
files: 1, // only allow one file per upload
fileSize: Uploader.getMaxFileSize(),
},
});
} catch (err) {
return abortWithError(err);
}

// Any error or limit from busboy will abort the upload with an error
busboyInstance.on("error", abortWithError);
busboyInstance.on("partsLimit", () => abortWithError(Error("Parts limit reached")));
busboyInstance.on("filesLimit", () => abortWithError(Error("Files limit reached")));
busboyInstance.on("fieldsLimit", () => abortWithError(Error("Fields limit reached")));

// generate a random output filename for the file
// we use do/while loop to prevent the rare case of generating a file name
// that already exists on disk
do {
randomName = crypto.randomBytes(8).toString("hex");
destDir = path.join(Config.getFileUploadPath(), randomName.substring(0, 2));
destPath = path.join(destDir, randomName);
} while (fs.existsSync(destPath));

// we split the filename into subdirectories (by taking 2 letters from the beginning)
// this helps avoid file system and certain tooling limitations when there are
// too many files on one folder
try {
fs.mkdirSync(destDir, {recursive: true});
} catch (err: any) {
log.error(`Error ensuring ${destDir} exists for uploads: ${err.message}`);
next();
}

return abortWithError(err);
static uploadMiddleware(this: void, req: Request, res: Response, next: NextFunction) {
if (req._username === undefined) {
return res.status(400).json({ error: "Upload token has no associated user" });
}

// Open a file stream for writing
streamWriter = fs.createWriteStream(destPath);
streamWriter.on("error", abortWithError);

busboyInstance.on(
"file",
(
fieldname: any,
fileStream: {
on: (
arg0: string,
arg1: {(err: any): Response<any, Record<string, any>>; (): void}
) => void;
unpipe: (arg0: any) => void;
read: {bind: (arg0: any) => any};
pipe: (arg0: any) => void;
},
filename: string | number | boolean
) => {
uploadUrl = `${randomName}/${encodeURIComponent(filename)}`;

if (Config.values.fileUpload.baseUrl) {
uploadUrl = new URL(uploadUrl, Config.values.fileUpload.baseUrl).toString();
} else {
uploadUrl = `uploads/${uploadUrl}`;
}

// if the busboy data stream errors out or goes over the file size limit
// abort the processing with an error
// @ts-expect-error Argument of type '(err: any) => Response<any, Record<string, any>>' is not assignable to parameter of type '{ (err: any): Response<any, Record<string, any>>; (): void; }'.ts(2345)
fileStream.on("error", abortWithError);
fileStream.on("limit", () => {
fileStream.unpipe(streamWriter);
fileStream.on("readable", fileStream.read.bind(fileStream));

return abortWithError(Error("File size limit reached"));
});

// Attempt to write the stream to file
fileStream.pipe(streamWriter);
}
);
const username = req._username;
const uploadPath = Config.getFileUploadPath();

busboyInstance.on("finish", () => {
doneCallback();
const userDir = resolvePath(uploadPath, username);

const uploadMiddleware = multer({
limits: {
files: 1,
fileSize: Uploader.getMaxFileSize()
},
storage: multer.diskStorage({
destination(_req, file, cb) {
fs.mkdir(userDir, { recursive: true }).then(() => {
cb(null, userDir);
}).catch((err) => {
log.error("File upload error: %s", err.message);
cb(err, "");
});
},
filename(_req, file, cb) {
let id = nanoid(16);
const ext = path.parse(file.originalname).ext;

if (!uploadUrl) {
return res.status(400).json({error: "Missing file"});
}
if (ext) {
id += ext;
}

// upload was done, send the generated file url to the client
res.status(200).json({
url: uploadUrl,
});
cb(null, id);
}
})
});

// pipe request body to busboy for processing
return req.pipe(busboyInstance);
uploadMiddleware.single("file")(req, res, next);
}

static afterUpload(this: void, req: Request, res: Response) {
if (req.file) {
log.info("uploaded file: '%s'", req.file.originalname);
}
}

static getMaxFileSize() {
Expand Down
2 changes: 1 addition & 1 deletion server/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ function initializeClient(
}

if (Config.values.fileUpload.enable) {
new Uploader(socket);
new Uploader(client, socket);
}

socket.on("disconnect", function () {
Expand Down
Loading

0 comments on commit 4b0ad89

Please sign in to comment.