diff --git a/Cargo.lock b/Cargo.lock index 602fd87..b7b53b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2586,7 +2586,7 @@ dependencies = [ [[package]] name = "rustus" -version = "0.4.1" +version = "0.4.2" dependencies = [ "actix-files", "actix-rt", diff --git a/Cargo.toml b/Cargo.toml index a083d0d..c9c9488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustus" -version = "0.4.1" +version = "0.4.2" edition = "2021" description = "TUS protocol implementation written in Rust." diff --git a/README.md b/README.md index ea1bb82..db7914b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ -# Rustus - -[Tus](https://tus.io/) protocol implementation written in Rust. +
+ logo +
+

+ Docker Image Size (latest by date) + Docker Image Version (latest semver) + GitHub +
+

TUS protocol implementation written in Rust.

+
## Features @@ -17,11 +24,13 @@ This implementation has several features to make usage as simple as possible. You can download binaries from a [releases page](https://github.com/s3rius/rustus/releases). If you want to use docker, you can use official images from [s3rius/rustus](https://hub.docker.com/r/s3rius/rustus/): + ```bash docker run --rm -it -p 1081:1081 s3rius/rustus:latest ``` -If we don't have a binary file for your operating system you can build it with [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html). +If we don't have a binary file for your operating system you can build it +with [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html). ```bash git clone https://github.com/s3rius/rustus.git @@ -31,8 +40,8 @@ cargo install --path . --features=all ### Supported data storages -Right now you can only use `file-storage` to store uploads data. -The only two options you can adjust are: +Right now you can only use `file-storage` to store uploads data. The only two options you can adjust are: + * uploads directory * directory structure @@ -43,8 +52,8 @@ you can provide a `--data-dir` parameter. rustus --data-dir "./files" ``` -If you have a lot of uploads, you don't want to store all your files in -a flat structure. So you can set a directory structure for your uploads. +If you have a lot of uploads, you don't want to store all your files in a flat structure. So you can set a directory +structure for your uploads. ```bash rustus --dir-structure="{env[HOSTNAME]}/{year}/{month}/{day}" @@ -64,21 +73,18 @@ data ``` -**Important note:** if you use variable that doesn't exist or incorrect like invalid env variable, it -results in an error and the directory structure will become flat again. +**Important note:** if you use variable that doesn't exist or incorrect like invalid env variable, it results in an +error and the directory structure will become flat again. -As you can see all info files are stored in a flat structure. It cannot be changed if -you use file info storage. In order to get rid of those `.info` files use different -info storages. +As you can see all info files are stored in a flat structure. It cannot be changed if you use file info storage. In +order to get rid of those `.info` files use different info storages. ## Info storages -The info storage is a database or directory. The main goal is to keep track -of uploads. Rustus stores information about download in json format inside -database. +The info storage is a database or directory. The main goal is to keep track of uploads. Rustus stores information about +download in json format inside database. -File storage is used by default. You can customize the directory of an .info files -by providing `--info-dir` parameter. +File storage is used by default. You can customize the directory of an .info files by providing `--info-dir` parameter. ```bash rustus --info-dir "./info_dir" @@ -100,24 +106,24 @@ rustus --info-storage db-info-storage --info-db-dsn "mysql://rustus:rustus@192.1 ## Hooks Rustus supports several event hooks, such as: + * File hooks; * HTTP hooks; * AMQP hooks. -You can combine them, but you have to be careful, since -AMQP hooks won't block uploading. +You can combine them, but you have to be careful, since AMQP hooks won't block uploading. -If you want to check the "Authorization" header value or validate some information, -you have to use webhooks or File hooks. +If you want to check the "Authorization" header value or validate some information, you have to use webhooks or File +hooks. -Hooks have priorities: file hooks are the most important, then goes webhooks and AMQP hooks have the least priority. -If pre-create hook failed, the upload would not start. -Of course, since AMQP is a protocol that doesn't allow you to track responses -we can't validate anything to stop uploading. +Hooks have priorities: file hooks are the most important, then goes webhooks and AMQP hooks have the least priority. If +pre-create hook failed, the upload would not start. Of course, since AMQP is a protocol that doesn't allow you to track +responses we can't validate anything to stop uploading. Hooks can have 2 formats default: + ```json { "upload": { @@ -152,6 +158,7 @@ default: ``` tusd: + ```json { "Upload": { @@ -215,9 +222,8 @@ Rustus can work with two types of file hooks. 1. Single file hook; 2. Hooks directory. -The main difference is that hook name is passed as a command line parameter to a -single file hook, but if you use hooks directory then hook name is used to determine a -file to call. Let's take a look at the examples +The main difference is that hook name is passed as a command line parameter to a single file hook, but if you use hooks +directory then hook name is used to determine a file to call. Let's take a look at the examples Example of a single file hook: @@ -239,11 +245,13 @@ fi As you can see it uses first CLI parameter as a hook name and all hook data is received from stdin. Let's make it executable + ```bash chmod +x "hooks/unified_hook" ``` To use it you can add parameter + ```bash rustus --hooks-file "hooks/unified_hook" ``` @@ -262,29 +270,25 @@ hooks └── pre-create ``` -Every file in this directory has an executable flag. -So you can specify a parameter to use hooks directory. +Every file in this directory has an executable flag. So you can specify a parameter to use hooks directory. ```bash rustus --hooks-dir "hooks" ``` -In this case rustus will append a hook name to the directory you pointed at and call it as -an executable. +In this case rustus will append a hook name to the directory you pointed at and call it as an executable. Information about hook is passed as a first parameter, as if you call script by running: + ```bash ./hooks/pre-create '{"id": "someid", ...}' ``` ### Http Hooks -Http hooks use http protocol to notify you about an upload. -You can use HTTP hooks to verify Authorization. +Http hooks use http protocol to notify you about an upload. You can use HTTP hooks to verify Authorization. - -Let's create a FastAPI application that listens to hooks and checks the -authorization header. +Let's create a FastAPI application that listens to hooks and checks the authorization header. ```bash # Installing dependencies @@ -298,10 +302,11 @@ from typing import Optional app = FastAPI() + @app.post("/hooks") def hook( - authorization: Optional[str] = Header(None), - hook_name: Optional[str] = Header(None), + authorization: Optional[str] = Header(None), + hook_name: Optional[str] = Header(None), ): print(f"Received: {hook_name}") if authorization != "Bearer jwt": @@ -310,16 +315,17 @@ def hook( ``` Now we can start a server. + ```bash uvicorn server:app --port 8080 ``` Now you can start rustus, and it will check if Authorization header has a correct value. + ```bash rustus --hooks-http-urls "http://localhost:8000/hooks" --hooks-http-proxy-headers "Authorization" ``` - ### AMQP hooks All hooks can be sent with an AMQP protocol. @@ -335,6 +341,5 @@ This command will create an exchange called "rustus" and queues for every hook. Every hook is published with routing key "rustus.{hook_name}" like "rustus.post-create" or "rustus.pre-create" and so on. -The problem with AMQP hooks is that you can't block the upload. -To do this you have to use HTTP or File hooks. But with AMQP your -uploads become non-blocking which is definitely a good thing. \ No newline at end of file +The problem with AMQP hooks is that you can't block the upload. To do this you have to use HTTP or File hooks. But with +AMQP your uploads become non-blocking which is definitely a good thing. \ No newline at end of file diff --git a/imgs/logo.svg b/imgs/logo.svg new file mode 100644 index 0000000..b1689a7 --- /dev/null +++ b/imgs/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/imgs/rustus_startup_logo.txt b/imgs/rustus_startup_logo.txt index 381d6f4..098a498 100644 --- a/imgs/rustus_startup_logo.txt +++ b/imgs/rustus_startup_logo.txt @@ -1,7 +1,7 @@ - ____ __ - / __ \__ _______/ /___ _______ - / /_/ / / / / ___/ __/ / / / ___/ - / _, _/ /_/ (__ ) /_/ /_/ (__ ) - /_/ |_|\__,_/____/\__/\__,_/____/ + __ + _______ _______/ /___ _______ + / ___/ / / / ___/ __/ / / / ___/ + / / / /_/ (__ ) /_/ /_/ (__ ) +/_/ \__,_/____/\__/\__,_/____/ Makes file uploads easier. diff --git a/src/info_storages/file_info_storage.rs b/src/info_storages/file_info_storage.rs index 65cd3ca..bafa5bc 100644 --- a/src/info_storages/file_info_storage.rs +++ b/src/info_storages/file_info_storage.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; use async_std::fs::{read_to_string, remove_file, DirBuilder, OpenOptions}; -use async_std::prelude::*; use async_trait::async_trait; +use futures::io::BufWriter; +use futures::AsyncWriteExt; use log::error; use crate::errors::{RustusError, RustusResult}; @@ -35,7 +36,7 @@ impl InfoStorage for FileInfoStorage { } async fn set_info(&self, file_info: &FileInfo, create: bool) -> RustusResult<()> { - let mut file = OpenOptions::new() + let file = OpenOptions::new() .write(true) .create(create) .truncate(true) @@ -45,7 +46,16 @@ impl InfoStorage for FileInfoStorage { error!("{:?}", err); RustusError::UnableToWrite(err.to_string()) })?; - file.write_all(serde_json::to_string(&file_info)?.as_bytes()) + let mut writer = BufWriter::new(file); + writer + .write_all( + serde_json::to_string(&file_info) + .map_err(|err| { + error!("{:#?}", err); + err + })? + .as_bytes(), + ) .await .map_err(|err| { error!("{:?}", err); @@ -56,7 +66,7 @@ impl InfoStorage for FileInfoStorage { .to_string(), ) })?; - file.sync_data().await?; + writer.flush().await?; Ok(()) } diff --git a/src/storages/file_storage.rs b/src/storages/file_storage.rs index ccc699c..ba44819 100644 --- a/src/storages/file_storage.rs +++ b/src/storages/file_storage.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use actix_files::NamedFile; use async_std::fs::{remove_file, DirBuilder, File, OpenOptions}; use async_std::io::copy; -use async_std::prelude::*; use async_trait::async_trait; use log::error; @@ -12,6 +11,8 @@ use crate::info_storages::FileInfo; use crate::storages::Storage; use crate::utils::dir_struct::dir_struct; use derive_more::Display; +use futures::io::BufWriter; +use futures::AsyncWriteExt; #[derive(Display)] #[display(fmt = "file_storage")] @@ -86,7 +87,7 @@ impl Storage for FileStorage { // Opening file in w+a mode. // It means that we're going to append some // bytes to the end of a file. - let mut file = OpenOptions::new() + let file = OpenOptions::new() .write(true) .append(true) .create(false) @@ -96,12 +97,13 @@ impl Storage for FileStorage { error!("{:?}", err); RustusError::UnableToWrite(err.to_string()) })?; - file.write_all(bytes).await.map_err(|err| { + let mut writer = BufWriter::new(file); + writer.write_all(bytes).await.map_err(|err| { error!("{:?}", err); RustusError::UnableToWrite(info.path.clone().unwrap()) })?; - file.sync_data().await?; // Updating information about file. + writer.flush().await?; Ok(()) } @@ -109,9 +111,10 @@ impl Storage for FileStorage { // New path to file. let file_path = self.data_file_path(file_info.id.as_str()).await?; // Creating new file. - let mut file = OpenOptions::new() - .write(true) + OpenOptions::new() .create(true) + .write(true) + .truncate(true) .create_new(true) .open(file_path.as_path()) .await @@ -119,15 +122,6 @@ impl Storage for FileStorage { error!("{:?}", err); RustusError::FileAlreadyExists(file_info.id.clone()) })?; - - // Let's write an empty string to the beginning of the file. - // Maybe remove it later. - file.write_all(b"").await.map_err(|err| { - error!("{:?}", err); - RustusError::UnableToWrite(file_path.display().to_string()) - })?; - file.sync_all().await?; - Ok(file_path.display().to_string()) }