Merge branch 'release/0.6.0'

Cargo.toml

@@ -1,13 +1,9 @@
 [package]
 name = "rustus"
-version = "0.5.14"
+version = "0.6.0"
 edition = "2021"
 description = "TUS protocol implementation written in Rust."
-keywords = [
-    "tus",
-    "server",
-    "actix-web",
-]
+keywords = ["tus", "server", "actix-web"]
 license-file = "LICENSE"
 authors = [
     "Pavel Kirilin <[email protected]>",
@@ -21,9 +17,9 @@ readme = "README.md"
 name = "rustus"
 
 [dependencies]
-bytes = "~1.2.1"
+bytes = "~1.3.0"
 async-trait = "^0.1.52"
-base64 = "^0.13.0"
+base64 = "0.20.0"
 log = "^0.4.14"
 serde_json = "^1"
 thiserror = "^1.0"
@@ -33,25 +29,18 @@ actix-web-prom = "^0.6.0"
 dyn-clone = "^1.0.5"
 actix-cors = "0.6.1"
 wildmatch = "2.1.0"
+md-5 = "^0.10.1"
+digest = "0.10.3"
 mimalloc = { version = "~0.1.30", default-features = false }
 
-[dependencies.digest]
-version = "0.10.3"
-optional = true
 
 [dependencies.sha1]
 version = "^0.10.1"
 features = ["compress"]
-optional = true
 
 [dependencies.sha2]
 version = "^0.10.1"
 features = ["compress"]
-optional = true
-
-[dependencies.md-5]
-version = "^0.10.1"
-optional = true
 
 [dependencies.futures]
 version = "^0.3.21"
@@ -107,7 +96,6 @@ version = "^2.0"
 
 [dependencies.reqwest]
 features = ["json"]
-optional = true
 version = "^0.11.8"
 
 [dependencies.structopt]
@@ -118,7 +106,17 @@ features = ["derive"]
 version = "0.24.0"
 
 [dependencies.tokio]
-features = ["time", "process", "fs", "io-std", "io-util", "rt-multi-thread", "bytes", "rt", "macros"]
+features = [
+    "time",
+    "process",
+    "fs",
+    "io-std",
+    "io-util",
+    "rt-multi-thread",
+    "bytes",
+    "rt",
+    "macros",
+]
 version = "^1.4.0"
 
 [dependencies.tokio-amqp]
@@ -129,14 +127,19 @@ version = "^1.0.0"
 features = ["v4"]
 version = "^1.0.0-alpha.1"
 
+[dependencies.rust-s3]
+version = "~0.32.3"
+
 [features]
-all = ["redis_info_storage", "db_info_storage", "http_notifier", "amqp_notifier", "hashers"]
+all = [
+    "redis_info_storage",
+    "db_info_storage",
+    "amqp_notifier",
+]
 amqp_notifier = ["lapin", "tokio-amqp", "mobc-lapin"]
 db_info_storage = ["rbatis", "rbson"]
 default = []
-http_notifier = ["reqwest"]
 redis_info_storage = ["mobc-redis"]
-hashers = ["md-5", "sha1", "sha2", "digest"]
 
 ### For testing
 test_redis = []

README.md

@@ -27,8 +27,8 @@ You can install rustus by 4 different ways.
 
 ### From source
 
-To build it from source rust must be installed.
-Preferred version is 1.59.0.
+To build it from source rust must be installed. We don't rely on nightly features,
+you can use last stable release.
 
 ```bash
 git clone https://github.com/s3rius/rustus.git
@@ -41,9 +41,7 @@ Available features:
 
 * `amqp_notifier` - adds amqp protocol support for notifying about upload status;
 * `db_info_storage` - adds support for storing information about upload in different databases (Postgres, MySQL, SQLite);
-* `http_notifier` - adds support for notifying about upload status via http protocol;
 * `redis_info_storage` - adds support for storing information about upload in redis database;
-* `hashers` - adds support for checksum verification;
 * `all` - enables all rustus features.
 
 All precompiled binaries have all features enabled.

deploy/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.64.0-bullseye AS builder
+FROM rust:1.66.0-bullseye AS builder
 
 WORKDIR /app
 COPY Cargo.toml Cargo.lock ./

docs/configuration.md

@@ -27,6 +27,8 @@ You can define which hosts are allowed for your particular application.
 For example if you add `--cors "*.staging.domain,*.prod.domain"`, it allows all origins
 like `my.staging.domain` or `my.prod.domain`, but it will refuse to serve other origins.
 
+Also you can disable access log for `/health` endpoint, by using `--disable-health-access-log`.
+
 === "CLI"
 
     ``` bash
@@ -35,8 +37,9 @@ like `my.staging.domain` or `my.prod.domain`, but it will refuse to serve other
         --workers 8 \
         --max-body-size 1000000 \
         --url "/files" \
-        --log-level "INFO"
-        --cors "my.*.domain.com,your.*.domain.com"
+        --log-level "INFO" \
+        --cors "my.*.domain.com,your.*.domain.com" \
+        --disable-health-access-log "yes"
     ```
 
 === "ENV"
@@ -49,22 +52,29 @@ like `my.staging.domain` or `my.prod.domain`, but it will refuse to serve other
     export RUSTUS_URL="/files"
     export RUSTUS_LOG_LEVEL="INFO"
     export RUSTUS_CORS="my.*.domain.com,your.*.domain.com"
+    export RUSTUS_DISABLE_HEALTH_ACCESS_LOG="yes"
 
     rustus
     ```
 
 
-## Configuring data storage
 
 
-!!!info
+## Configuring storage
+
+Storages are used to actually store your files. You can configure where you want
+to store files. By default in uses `file-storage` and stores every upload on
+your local file system.
+
+Availabe storages:
+
+* `file-storage`
+* `hybrid-s3`
 
-    Currently only file storage is available,
-    so if you pass to `--storage` parameter other than `file-storage` you will get an error.
+### File storage
 
-Available parameters:
+File storage parameters:
 
-* `--storage` - type of data storage to be used;
 * `--data-dir` - path to the directory where all files are stored;
 * `--dir-structure` - pattern of a directory structure inside data dir;
 * `--force-fsync` - calls fsync system call after every write to disk.
@@ -112,6 +122,78 @@ data
     rustus
     ```
 
+### Hybrid-S3 storage
+
+This storage stores files locally and uploads resulting file on S3 when the upload is finished.
+It has no restriction on chunk size and you can make chunks less than 5MB.
+
+!!! Danger
+    When choosing this storage you still need to have a
+    connected shared directory between instances.
+
+    This storage is not intended to be used for large files,
+    since it uploads files to S3 during the last request.
+
+Hybrid-S3 uses file-storage inside, so all parameters from file storage
+also applied to it.
+
+Parameters:
+
+* `--dir-structure` - pattern of a directory structure locally and on s3;
+* `--data-dir` - path to the local directory where all files are stored;
+* `--force-fsync` - calls fsync system call after every write to disk in local storage;
+* `--s3-url` -  s3 endpoint URL;
+* `--s3-bucket` - name of a bucket to use;
+* `--s3-region` - AWS region to use;
+* `--s3-access-key` - S3 access key;
+* `--s3-secret-key` - S3 secret key;
+* `--s3-security-token` - s3 secrity token;
+* `--s3-session-token` - S3 session token;
+* `--s3-profile` - Name of the section from `~/.aws/credentials` file;
+* `--s3-headers` - JSON object with additional header to every S3 request (Useful for setting ACLs);
+* `--s3-force-path-style` - use path style URL. It appends bucket name at the end of the URL;
+
+Required parameter are only `--s3-url` and `--s3-bucket`.
+
+=== "CLI"
+
+    ``` bash
+    rustus --storage "hybrid-s3" \
+        --s3-url "https://localhost:9000" \
+        --s3-bucket "bucket" \
+        --s3-region "eu-central1" \
+        --s3-access-key "fJljHcXo07rqIOzh" \
+        --s3-secret-key "6BJfBUL18nLiGmF5zKW0NKrdxQVxNYWB" \
+        --s3-profile "my_profile" \
+        --s3-security-token "token" \
+        --s3-session-token "token" \
+        --s3-force-path-style "yes" \
+        --s3-headers '{"x-amz-acl": "public-read"}' \
+        --force-fsync "yes" \
+        --data-dir "./data/" \
+        --dir-structure "{year}/{month}/{day}"
+    ```
+
+=== "ENV"
+
+    ``` bash
+    export RUSTUS_STORAGE="hybrid-s3"
+    export RUSTUS_S3_URL="https://localhost:9000"
+    export RUSTUS_S3_BUCKET="bucket"
+    export RUSTUS_S3_REGION="eu-central1"
+    export RUSTUS_S3_ACCESS_KEY="fJljHcXo07rqIOzh"
+    export RUSTUS_S3_SECRET_KEY="6BJfBUL18nLiGmF5zKW0NKrdxQVxNYWB"
+    export RUSTUS_S3_SECURITY_TOKEN="token"
+    export RUSTUS_S3_SESSION_TOKEN="token"
+    export RUSTUS_S3_PROFILE="my_profile"
+    export RUSTUS_S3_HEADERS='{"x-amz-acl": "public-read"}'
+    export RUSTUS_DATA_DIR="./data/"
+    export RUSTUS_DIR_STRUCTURE="{year}/{month}/{day}"
+    export RUSTUS_FORCE_FSYNC="yes"
+
+    rustus
+    ```
+
 ## Configuring info storage
 
 Info storages are used to store information

docs/index.md

@@ -24,13 +24,13 @@ You can install rustus in four different ways.
 
 ### From source
 
-To build it from source rust must be installed.
-Preferred version is 1.59.0.
+To build it from source rust must be installed. We don't rely on nightly features,
+you can use last stable release.
 
 ```bash
 git clone https://github.com/s3rius/rustus.git
 cd rustus
-cargo install --path . --features=all,metrics
+cargo install --path . --features=all
 ```
 
 Also, you can speedup build by disabling some features.
@@ -39,9 +39,7 @@ Available features:
 
 * `amqp_notifier` - adds `AMQP` protocol support for notifying about upload status;
 * `db_info_storage` - adds support for storing information about upload in different databases (`Postgres`, `MySQL`, `SQLite`);
-* `http_notifier` - adds support for notifying about upload status via `HTTP` protocol;
 * `redis_info_storage` - adds support for storing information about upload in `Redis` database;
-* `hashers` - adds support for checksum verification;
 * `all` - enables all rustus features.
 
 All precompiled binaries have all features enabled.

src/config.rs

@@ -43,6 +43,68 @@ pub struct StorageOptions {
     /// In most cases this parameter is redundant.
     #[structopt(long, env = "RUSTUS_FORCE_FSYNC")]
     pub force_fsync: bool,
+
+    /// S3 bucket to upload files to.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_BUCKET")]
+    pub s3_bucket: Option<String>,
+
+    /// S3 region.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_REGION")]
+    pub s3_region: Option<String>,
+
+    /// S3 access key.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_ACCESS_KEY")]
+    pub s3_access_key: Option<String>,
+
+    /// S3 secret key.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SECRET_KEY")]
+    pub s3_secret_key: Option<String>,
+
+    /// S3 URL.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, required_if("storage", "hybrid-s3"), env = "RUSTUS_S3_URL")]
+    pub s3_url: Option<String>,
+
+    /// S3 force path style.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_FORCE_PATH_STYLE")]
+    pub s3_force_path_style: bool,
+
+    /// S3 security token.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SECURITY_TOKEN")]
+    pub s3_security_token: Option<String>,
+
+    /// S3 session token.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_SESSION_TOKEN")]
+    pub s3_session_token: Option<String>,
+
+    /// S3 profile.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_PROFILE")]
+    pub s3_profile: Option<String>,
+
+    /// Additional S3 headers.
+    /// These headers are passed to every request to s3.
+    /// Useful for configuring ACLs.
+    ///
+    /// This parameter is required fo s3-based storages.
+    #[structopt(long, env = "RUSTUS_S3_HEADERS")]
+    pub s3_headers: Option<String>,
 }
 
 #[derive(StructOpt, Debug, Clone)]
@@ -112,12 +174,10 @@ pub struct NotificationsOptions {
     pub behind_proxy: bool,
 
     /// List of URLS to send webhooks to.
-    #[cfg(feature = "http_notifier")]
     #[structopt(long, env = "RUSTUS_HOOKS_HTTP_URLS", use_delimiter = true)]
     pub hooks_http_urls: Vec<String>,
 
     // List of headers to forward from client.
-    #[cfg(feature = "http_notifier")]
     #[structopt(long, env = "RUSTUS_HOOKS_HTTP_PROXY_HEADERS", use_delimiter = true)]
     pub hooks_http_proxy_headers: Vec<String>,
 
@@ -205,6 +265,9 @@ pub struct RustusConf {
     #[structopt(short, long, default_value = "1081", env = "RUSTUS_SERVER_PORT")]
     pub port: u16,
 
+    #[structopt(long, env = "RUSTUS_DISABLE_HEALTH_ACCESS_LOG")]
+    pub disable_health_access_log: bool,
+
     /// Rustus base API url
     #[structopt(long, default_value = "/files", env = "RUSTUS_URL")]
     pub url: String,