Feature/s3 support (#108)

* Added Hybrid-S3 storage.

Signed-off-by: Pavel Kirilin <[email protected]>

Cargo.toml

@@ -3,11 +3,7 @@ name = "rustus"
 version = "0.5.14"
 edition = "2021"
 description = "TUS protocol implementation written in Rust."
-keywords = [
-    "tus",
-    "server",
-    "actix-web",
-]
+keywords = ["tus", "server", "actix-web"]
 license-file = "LICENSE"
 authors = [
     "Pavel Kirilin <[email protected]>",
@@ -21,9 +17,9 @@ readme = "README.md"
 name = "rustus"
 
 [dependencies]
-bytes = "~1.2.1"
+bytes = "~1.3.0"
 async-trait = "^0.1.52"
-base64 = "^0.13.0"
+base64 = "0.20.0"
 log = "^0.4.14"
 serde_json = "^1"
 thiserror = "^1.0"
@@ -33,25 +29,18 @@ actix-web-prom = "^0.6.0"
 dyn-clone = "^1.0.5"
 actix-cors = "0.6.1"
 wildmatch = "2.1.0"
+md-5 = "^0.10.1"
+digest = "0.10.3"
 mimalloc = { version = "~0.1.30", default-features = false }
 
-[dependencies.digest]
-version = "0.10.3"
-optional = true
 
 [dependencies.sha1]
 version = "^0.10.1"
 features = ["compress"]
-optional = true
 
 [dependencies.sha2]
 version = "^0.10.1"
 features = ["compress"]
-optional = true
-
-[dependencies.md-5]
-version = "^0.10.1"
-optional = true
 
 [dependencies.futures]
 version = "^0.3.21"
@@ -107,7 +96,6 @@ version = "^2.0"
 
 [dependencies.reqwest]
 features = ["json"]
-optional = true
 version = "^0.11.8"
 
 [dependencies.structopt]
@@ -118,7 +106,17 @@ features = ["derive"]
 version = "0.24.0"
 
 [dependencies.tokio]
-features = ["time", "process", "fs", "io-std", "io-util", "rt-multi-thread", "bytes", "rt", "macros"]
+features = [
+    "time",
+    "process",
+    "fs",
+    "io-std",
+    "io-util",
+    "rt-multi-thread",
+    "bytes",
+    "rt",
+    "macros",
+]
 version = "^1.4.0"
 
 [dependencies.tokio-amqp]
@@ -129,14 +127,19 @@ version = "^1.0.0"
 features = ["v4"]
 version = "^1.0.0-alpha.1"
 
+[dependencies.rust-s3]
+version = "~0.32.3"
+
 [features]
-all = ["redis_info_storage", "db_info_storage", "http_notifier", "amqp_notifier", "hashers"]
+all = [
+    "redis_info_storage",
+    "db_info_storage",
+    "amqp_notifier",
+]
 amqp_notifier = ["lapin", "tokio-amqp", "mobc-lapin"]
 db_info_storage = ["rbatis", "rbson"]
 default = []
-http_notifier = ["reqwest"]
 redis_info_storage = ["mobc-redis"]
-hashers = ["md-5", "sha1", "sha2", "digest"]
 
 ### For testing
 test_redis = []

README.md

@@ -41,9 +41,7 @@ Available features:
 
 * `amqp_notifier` - adds amqp protocol support for notifying about upload status;
 * `db_info_storage` - adds support for storing information about upload in different databases (Postgres, MySQL, SQLite);
-* `http_notifier` - adds support for notifying about upload status via http protocol;
 * `redis_info_storage` - adds support for storing information about upload in redis database;
-* `hashers` - adds support for checksum verification;
 * `all` - enables all rustus features.
 
 All precompiled binaries have all features enabled.

docs/configuration.md

@@ -58,17 +58,23 @@ Also you can disable access log for `/health` endpoint, by using `--disable-heal
     ```
 
 
-## Configuring data storage
 
 
-!!!info
+## Configuring storage
 
-    Currently only file storage is available,
-    so if you pass to `--storage` parameter other than `file-storage` you will get an error.
+Storages are used to actually store your files. You can configure where you want
+to store files. By default in uses `file-storage` and stores every upload on
+your local file system.
 
-Available parameters:
+Availabe storages:
+
+* `file-storage`
+* `hybrid-s3`
+
+### File storage
+
+File storage parameters:
 
-* `--storage` - type of data storage to be used;
 * `--data-dir` - path to the directory where all files are stored;
 * `--dir-structure` - pattern of a directory structure inside data dir;
 * `--force-fsync` - calls fsync system call after every write to disk.
@@ -116,6 +122,78 @@ data
     rustus
     ```
 
+### Hybrid-S3 storage
+
+This storage stores files locally and uploads resulting file on S3 when the upload is finished.
+It has no restriction on chunk size and you can make chunks less than 5MB.
+
+!!! Danger
+    When choosing this storage you still need to have a
+    connected shared directory between instances.
+
+    This storage is not intended to be used for large files,
+    since it uploads files to S3 during the last request.
+
+Hybrid-S3 uses file-storage inside, so all parameters from file storage
+also applied to it.
+
+Parameters:
+
+* `--dir-structure` - pattern of a directory structure locally and on s3;
+* `--data-dir` - path to the local directory where all files are stored;
+* `--force-fsync` - calls fsync system call after every write to disk in local storage;
+* `--s3-url` -  s3 endpoint URL;
+* `--s3-bucket` - name of a bucket to use;
+* `--s3-region` - AWS region to use;
+* `--s3-access-key` - S3 access key;
+* `--s3-secret-key` - S3 secret key;
+* `--s3-security-token` - s3 secrity token;
+* `--s3-session-token` - S3 session token;
+* `--s3-profile` - Name of the section from `~/.aws/credentials` file;
+* `--s3-headers` - JSON object with additional header to every S3 request (Useful for setting ACLs);
+* `--s3-force-path-style` - use path style URL. It appends bucket name at the end of the URL;
+
+Required parameter are only `--s3-url` and `--s3-bucket`.
+
+=== "CLI"
+
+    ``` bash
+    rustus --storage "hybrid-s3" \
+        --s3-url "https://localhost:9000" \
+        --s3-bucket "bucket" \
+        --s3-region "eu-central1" \
+        --s3-access-key "fJljHcXo07rqIOzh" \
+        --s3-secret-key "6BJfBUL18nLiGmF5zKW0NKrdxQVxNYWB" \
+        --s3-profile "my_profile" \
+        --s3-security-token "token" \
+        --s3-session-token "token" \
+        --s3-force-path-style "yes" \
+        --s3-headers '{"x-amz-acl": "public-read"}' \
+        --force-fsync "yes" \
+        --data-dir "./data/" \
+        --dir-structure "{year}/{month}/{day}"
+    ```
+
+=== "ENV"
+
+    ``` bash
+    export RUSTUS_STORAGE="hybrid-s3"
+    export RUSTUS_S3_URL="https://localhost:9000"
+    export RUSTUS_S3_BUCKET="bucket"
+    export RUSTUS_S3_REGION="eu-central1"
+    export RUSTUS_S3_ACCESS_KEY="fJljHcXo07rqIOzh"
+    export RUSTUS_S3_SECRET_KEY="6BJfBUL18nLiGmF5zKW0NKrdxQVxNYWB"
+    export RUSTUS_S3_SECURITY_TOKEN="token"
+    export RUSTUS_S3_SESSION_TOKEN="token"
+    export RUSTUS_S3_PROFILE="my_profile"
+    export RUSTUS_S3_HEADERS='{"x-amz-acl": "public-read"}'
+    export RUSTUS_DATA_DIR="./data/"
+    export RUSTUS_DIR_STRUCTURE="{year}/{month}/{day}"
+    export RUSTUS_FORCE_FSYNC="yes"
+
+    rustus
+    ```
+
 ## Configuring info storage
 
 Info storages are used to store information

docs/index.md

@@ -39,9 +39,7 @@ Available features:
 
 * `amqp_notifier` - adds `AMQP` protocol support for notifying about upload status;
 * `db_info_storage` - adds support for storing information about upload in different databases (`Postgres`, `MySQL`, `SQLite`);
-* `http_notifier` - adds support for notifying about upload status via `HTTP` protocol;
 * `redis_info_storage` - adds support for storing information about upload in `Redis` database;
-* `hashers` - adds support for checksum verification;
 * `all` - enables all rustus features.
 
 All precompiled binaries have all features enabled.

src/config.rs

@@ -43,6 +43,68 @@ pub struct StorageOptions {
     /// In most cases this parameter is redundant.
     #[structopt(long, env = "RUSTUS_FORCE_FSYNC")]
     pub force_fsync: bool,
+
+    /// S3 bucket to upload files to.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_BUCKET")]
+    pub s3_bucket: Option<String>,
+
+    /// S3 region.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_REGION")]
+    pub s3_region: Option<String>,
+
+    /// S3 access key.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_ACCESS_KEY")]
+    pub s3_access_key: Option<String>,
+
+    /// S3 secret key.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SECRET_KEY")]
+    pub s3_secret_key: Option<String>,
+
+    /// S3 URL.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_URL")]
+    pub s3_url: Option<String>,
+
+    /// S3 force path style.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_FORCE_PATH_STYLE")]
+    pub s3_force_path_style: bool,
+
+    /// S3 security token.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SECURITY_TOKEN")]
+    pub s3_security_token: Option<String>,
+
+    /// S3 session token.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SESSION_TOKEN")]
+    pub s3_session_token: Option<String>,
+
+    /// S3 profile.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_PROFILE")]
+    pub s3_profile: Option<String>,
+
+    /// Additional S3 headers.
+    /// These headers are passed to every request to s3.
+    /// Useful for configuring ACLs.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_HEADERS")]
+    pub s3_headers: Option<String>,
 }
 
 #[derive(StructOpt, Debug, Clone)]
@@ -112,12 +174,10 @@ pub struct NotificationsOptions {
     pub behind_proxy: bool,
 
     /// List of URLS to send webhooks to.
-    #[cfg(feature = "http_notifier")]
     #[structopt(long, env = "RUSTUS_HOOKS_HTTP_URLS", use_delimiter = true)]
     pub hooks_http_urls: Vec<String>,
 
     // List of headers to forward from client.
-    #[cfg(feature = "http_notifier")]
     #[structopt(long, env = "RUSTUS_HOOKS_HTTP_PROXY_HEADERS", use_delimiter = true)]
     pub hooks_http_proxy_headers: Vec<String>,
 

src/errors.rs

@@ -7,6 +7,8 @@ pub type RustusResult<T> = Result<T, RustusError>;
 
 #[derive(thiserror::Error, Debug)]
 pub enum RustusError {
+    #[error("{0}")]
+    Unimplemented(String),
     #[error("Not found")]
     FileNotFound,
     #[error("File already exists")]
@@ -42,7 +44,6 @@ pub enum RustusError {
     UnableToPrepareStorage(String),
     #[error("Unknown extension: {0}")]
     UnknownExtension(String),
-    #[cfg(feature = "http_notifier")]
     #[error("Http request failed: {0}")]
     HttpRequestError(#[from] reqwest::Error),
     #[error("Hook invocation failed. Reason: {0}")]
@@ -71,6 +72,8 @@ pub enum RustusError {
     BlockingError(#[from] actix_web::error::BlockingError),
     #[error("HTTP hook error. Returned status: {0}")]
     HTTPHookError(u16, String, Option<String>),
+    #[error("Found S3 error: {0}")]
+    S3Error(#[from] s3::error::S3Error),
 }
 
 /// This conversion allows us to use `RustusError` in the `main` function.
@@ -99,7 +102,7 @@ impl ResponseError for RustusError {
             }
             _ => HttpResponseBuilder::new(self.status_code())
                 .insert_header(("Content-Type", "text/html; charset=utf-8"))
-                .body(format!("{}", self)),
+                .body(format!("{self}")),
         }
     }
 

src/info_storages/file_info_storage.rs

@@ -27,7 +27,7 @@ impl FileInfoStorage {
     }
 
     pub fn info_file_path(&self, file_id: &str) -> PathBuf {
-        self.info_dir.join(format!("{}.info", file_id))
+        self.info_dir.join(format!("{file_id}.info"))
     }
 }
 

src/info_storages/models/file_info.rs

@@ -78,7 +78,7 @@ impl FileInfo {
         for (key, val) in &self.metadata {
             let encoded_value = base64::encode(val);
             // Adding metadata entry to the list.
-            result.push(format!("{} {}", key, encoded_value));
+            result.push(format!("{key} {encoded_value}"));
         }
 
         if result.is_empty() {