feat: config option object_storage.max_request_attempts (#6418)

* feat: config option object_storage.max_attempts Backblaze recommends to have a high amount of attempts since they've designed their architecture so that it will return 5xx errors to indicate that the client should do a new attempt. https://www.backblaze.com/blog/b2-503-500-server-error/ closes #6415 * Rephrase comment --------- Co-authored-by: Chocobozzz <[email protected]>
Chocobozzz · Jun 11, 2024 · ca26687 · ca26687
1 parent 46a2c15
commit ca26687
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 2 deletions.
diff --git a/config/default.yaml b/config/default.yaml
@@ -214,6 +214,11 @@ object_storage:
   # Maximum amount to upload in one request to object storage
   max_upload_part: 100MB
 
+  # Maximum number of attempts to make a request to object storage
+  # Some object storage providers (for instance Backblaze) expects the client to retry upload upon 5xx errors
+  # If you're using such a provider then you can increase this value
+  max_request_attempts: 3
+
   streaming_playlists:
     bucket_name: 'streaming-playlists'
 

diff --git a/config/production.yaml.example b/config/production.yaml.example
@@ -212,6 +212,11 @@ object_storage:
   # Maximum amount to upload in one request to object storage
   max_upload_part: 100MB
 
+  # Maximum number of attempts to make a request to object storage
+  # Some object storage providers (for instance Backblaze) expects the client to retry upload upon 5xx errors
+  # If you're using such a provider then you can increase this value
+  max_request_attempts: 3
+
   streaming_playlists:
     bucket_name: 'streaming-playlists'
 

diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts
@@ -67,7 +67,7 @@ function checkMissedConfig () {
     'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name',
     'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name',
     'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', 'object_storage.original_video_files.bucket_name',
-    'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url',
+    'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url', 'object_storage.max_request_attempts',
     'theme.default',
     'feeds.videos.count', 'feeds.comments.count',
     'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url',

diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts
@@ -133,6 +133,7 @@ const CONFIG = {
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
     MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
+    MAX_REQUEST_ATTEMPTS: config.get<number>('object_storage.max_request_attempts'),
     ENDPOINT: config.get<string>('object_storage.endpoint'),
     REGION: config.get<string>('object_storage.region'),
     UPLOAD_ACL: {

diff --git a/server/core/lib/object-storage/shared/client.ts b/server/core/lib/object-storage/shared/client.ts
@@ -45,7 +45,8 @@ function getClient () {
           secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
         }
         : undefined,
-      requestHandler: await getProxyRequestHandler()
+      requestHandler: await getProxyRequestHandler(),
+      maxAttempts: CONFIG.OBJECT_STORAGE.MAX_REQUEST_ATTEMPTS
     })
 
     logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())