🏗️(backends) integrate unified backends in the CLI

After unifying database and storage backends under a common interface, backends
settings are now handled directly alongside the backends classes.
Modifying the CLI to support new settings and new backends interfaces.

.env.dist

@@ -13,63 +13,90 @@ RALPH_APP_DIR=/app/.ralph
 # define them for convenience purpose during development, but they can be
 # passed as CLI options.
 
-# RALPH_BACKENDS__STORAGE__LDP__ENDPOINT=
-# RALPH_BACKENDS__STORAGE__LDP__APPLICATION_KEY=
-# RALPH_BACKENDS__STORAGE__LDP__APPLICATION_SECRET=
-# RALPH_BACKENDS__STORAGE__LDP__CONSUMER_KEY=
-# RALPH_BACKENDS__STORAGE__LDP__SERVICE_NAME=
-# RALPH_BACKENDS__STORAGE__LDP__STREAM_ID=
+# RALPH_BACKENDS__DATA__LDP__APPLICATION_KEY=
+# RALPH_BACKENDS__DATA__LDP__APPLICATION_SECRET=
+# RALPH_BACKENDS__DATA__LDP__CONSUMER_KEY=
+# RALPH_BACKENDS__DATA__LDP__DEFAULT_STREAM_ID=
+# RALPH_BACKENDS__DATA__LDP__ENDPOINT=
+# RALPH_BACKENDS__DATA__LDP__REQUEST_TIMEOUT=
+# RALPH_BACKENDS__DATA__LDP__SERVICE_NAME=
 
 # Swift storage backend
 
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_AUTH_URL=http://swift:35357/v3/
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_IDENTITY_API_VERSION=3
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_USER_DOMAIN_NAME=Default
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_PROJECT_DOMAIN_NAME=Default
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_TENANT_ID=cd238e84310a46e58af7f1d515887d88
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_TENANT_NAME=RegionOne
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_USERNAME=demo
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_PASSWORD=demo
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_REGION_NAME=RegionOne
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_STORAGE_URL=http://swift:8080/v1/KEY_cd238e84310a46e58af7f1d515887d88/test_container
+# RALPH_BACKENDS__DATA__SWIFT__AUTH_URL=http://swift:35357/v3/
+# RALPH_BACKENDS__DATA__SWIFT__USERNAME=demo
+# RALPH_BACKENDS__DATA__SWIFT__PASSWORD=demo
+# RALPH_BACKENDS__DATA__SWIFT__IDENTITY_API_VERSION=3
+# RALPH_BACKENDS__DATA__SWIFT__TENANT_ID=cd238e84310a46e58af7f1d515887d88
+# RALPH_BACKENDS__DATA__SWIFT__TENANT_NAME=RegionOne
+# RALPH_BACKENDS__DATA__SWIFT__PROJECT_DOMAIN_NAME=Default
+# RALPH_BACKENDS__DATA__SWIFT__REGION_NAME=RegionOne
+# RALPH_BACKENDS__DATA__SWIFT__OBJECT_STORAGE_URL=http://swift:8080/v1/KEY_cd238e84310a46e58af7f1d515887d88/test_container
+# RALPH_BACKENDS__DATA__SWIFT__USER_DOMAIN_NAME=Default
+# RALPH_BACKENDS__DATA__SWIFT__DEFAULT_CONTAINER=
+# RALPH_BACKENDS__DATA__SWIFT__LOCALE_ENCODING=Default
 
 # S3 storage backend
 
-# RALPH_BACKENDS__STORAGE__S3__ACCESS_KEY_ID=
-# RALPH_BACKENDS__STORAGE__S3__SECRET_ACCESS_KEY=
-# RALPH_BACKENDS__STORAGE__S3__SESSION_TOKEN=
-# RALPH_BACKENDS__STORAGE__S3__DEFAULT_REGION=
-# RALPH_BACKENDS__STORAGE__S3__BUCKET_NAME=
-# RALPH_BACKENDS__STORAGE__S3__ENDPOINT_URL=
+# RALPH_BACKENDS__DATA__S3__ACCESS_KEY_ID=
+# RALPH_BACKENDS__DATA__S3__SECRET_ACCESS_KEY=
+# RALPH_BACKENDS__DATA__S3__SESSION_TOKEN=
+# RALPH_BACKENDS__DATA__S3__ENDPOINT_URL=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_REGION=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_BUCKET_NAME=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_CHUNK_SIZE=
+# RALPH_BACKENDS__DATA__S3__LOCALE_ENCODING=
 
 # ES database backend
 
-RALPH_BACKENDS__DATABASE__ES__HOSTS=http://elasticsearch:9200
-RALPH_BACKENDS__DATABASE__ES__INDEX=statements
-RALPH_BACKENDS__DATABASE__ES__TEST_HOSTS=http://elasticsearch:9200
-RALPH_BACKENDS__DATABASE__ES__TEST_INDEX=test-index-foo
-RALPH_BACKENDS__DATABASE__ES__TEST_FORWARDING_INDEX=test-index-foo-2
+RALPH_BACKENDS__DATA__ES__HOSTS=http://elasticsearch:9200
+RALPH_BACKENDS__DATA__ES__DEFAULT_INDEX=statements
+# RALPH_BACKENDS__DATA__ES__ALLOW_YELLOW_STATUS=False
+# RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__ca_certs=False
+# RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs=False
+# RALPH_BACKENDS__DATA__ES__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__ES__LOCALE_ENCODING=utf8
+# RALPH_BACKENDS__DATA__ES__POINT_IN_TIME_KEEP_ALIVE=1m
+# RALPH_BACKENDS__DATA__ES__REFRESH_AFTER_WRITE=False
+RALPH_BACKENDS__DATA__ES__TEST_HOSTS=http://elasticsearch:9200
+RALPH_BACKENDS__DATA__ES__TEST_INDEX=test-index-foo
+RALPH_BACKENDS__DATA__ES__TEST_FORWARDING_INDEX=test-index-foo-2
 
 # MONGO database backend
 
-RALPH_BACKENDS__DATABASE__MONGO__COLLECTION=foo
-RALPH_BACKENDS__DATABASE__MONGO__DATABASE=statements
-RALPH_BACKENDS__DATABASE__MONGO__CONNECTION_URI=mongodb://mongo:27017/
-RALPH_BACKENDS__DATABASE__MONGO__TEST_COLLECTION=foo
-RALPH_BACKENDS__DATABASE__MONGO__TEST_FORWARDING_COLLECTION=foo-2
-RALPH_BACKENDS__DATABASE__MONGO__TEST_DATABASE=statements
-RALPH_BACKENDS__DATABASE__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
+RALPH_BACKENDS__DATA__MONGO__CONNECTION_URI=mongodb://mongo:27017/
+RALPH_BACKENDS__DATA__MONGO__DEFAULT_COLLECTION=foo
+RALPH_BACKENDS__DATA__MONGO__DEFAULT_DATABASE=statements
+# RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__document_class=
+# RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware=False
+# RALPH_BACKENDS__DATA__MONGO__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__MONGO__LOCALE_ENCODING=utf8
+RALPH_BACKENDS__DATA__MONGO__TEST_COLLECTION=foo
+RALPH_BACKENDS__DATA__MONGO__TEST_FORWARDING_COLLECTION=foo-2
+RALPH_BACKENDS__DATA__MONGO__TEST_DATABASE=statements
+RALPH_BACKENDS__DATA__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
 
 # ClickHouse database backend
 
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__HOST=clickhouse
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__PORT=8123
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__XAPI_DATABASE=xapi
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__EVENT_TABLE_NAME=xapi_events_all
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_DATABASE=test_statements
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_HOST=clickhouse
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_PORT=8123
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
+RALPH_BACKENDS__DATA__CLICKHOUSE__HOST=clickhouse
+RALPH_BACKENDS__DATA__CLICKHOUSE__PORT=8123
+RALPH_BACKENDS__DATA__CLICKHOUSE__DATABASE=xapi
+RALPH_BACKENDS__DATA__CLICKHOUSE__EVENT_TABLE_NAME=xapi_events_all
+# RALPH_BACKENDS__DATA__CLICKHOUSE__USERNAME=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__PASSWORD=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__CLIENT_OPTIONS__date_time_input_format=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__CLIENT_OPTIONS__allow_experimental_object_type=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__CLICKHOUSE__LOCALE_ENCODING=utf8
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_DATABASE=test_statements
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_HOST=clickhouse
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_PORT=8123
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
+
+
+# LRS HTTP backend
+
+RALPH_BACKENDS__HTTP__LRS__CONNECTION_URL=http://ralph:[email protected]:8100/
 
 # Sentry
 

src/ralph/backends/conf.py

@@ -0,0 +1,89 @@
+"""Configurations for Ralph backends."""
+
+from pydantic import BaseModel, BaseSettings
+
+from ralph.backends.data.clickhouse import ClickHouseDataBackendSettings
+from ralph.backends.data.es import ESDataBackendSettings
+from ralph.backends.data.fs import FSDataBackendSettings
+from ralph.backends.data.ldp import LDPDataBackendSettings
+from ralph.backends.data.mongo import MongoDataBackendSettings
+from ralph.backends.data.s3 import S3DataBackendSettings
+from ralph.backends.data.swift import SwiftDataBackendSettings
+from ralph.backends.http.async_lrs import LRSHTTPBackendSettings
+from ralph.backends.lrs.clickhouse import ClickHouseLRSBackendSettings
+from ralph.backends.lrs.fs import FSLRSBackendSettings
+from ralph.backends.stream.ws import WSStreamBackendSettings
+from ralph.conf import BaseSettingsConfig, core_settings
+
+# Active Data backend Settings.
+
+
+class DataBackendSettings(BaseModel):
+    """Pydantic model for data backend configuration settings."""
+
+    ASYNC_ES: ESDataBackendSettings = ESDataBackendSettings()
+    CLICKHOUSE: ClickHouseDataBackendSettings = ClickHouseDataBackendSettings()
+    ES: ESDataBackendSettings = ESDataBackendSettings()
+    FS: FSDataBackendSettings = FSDataBackendSettings()
+    LDP: LDPDataBackendSettings = LDPDataBackendSettings()
+    MONGO: MongoDataBackendSettings = MongoDataBackendSettings()
+    SWIFT: SwiftDataBackendSettings = SwiftDataBackendSettings()
+    S3: S3DataBackendSettings = S3DataBackendSettings()
+
+
+# Active HTTP backend Settings.
+
+
+class HTTPBackendSettings(BaseModel):
+    """Pydantic model for HTTP backend configuration settings."""
+
+    LRS: LRSHTTPBackendSettings = LRSHTTPBackendSettings()
+
+
+# Active LRS backend Settings.
+
+
+class LRSBackendSettings(BaseModel):
+    """Pydantic model for LRS backend configuration settings."""
+
+    ASYNC_ES: ESDataBackendSettings = ESDataBackendSettings()
+    CLICKHOUSE: ClickHouseLRSBackendSettings = ClickHouseLRSBackendSettings()
+    ES: ESDataBackendSettings = ESDataBackendSettings()
+    FS: FSLRSBackendSettings = FSLRSBackendSettings()
+    MONGO: MongoDataBackendSettings = MongoDataBackendSettings()
+
+
+# Active Stream backend Settings.
+
+
+class StreamBackendSettings(BaseModel):
+    """Pydantic model for stream backend configuration settings."""
+
+    WS: WSStreamBackendSettings = WSStreamBackendSettings()
+
+
+# Active backend Settings.
+
+
+class Backends(BaseModel):
+    """Pydantic model for backends configuration settings."""
+
+    DATA: DataBackendSettings = DataBackendSettings()
+    HTTP: HTTPBackendSettings = HTTPBackendSettings()
+    LRS: LRSBackendSettings = LRSBackendSettings()
+    STREAM: StreamBackendSettings = StreamBackendSettings()
+
+
+class BackendSettings(BaseSettings):
+    """Pydantic model for Ralph's backends environment & configuration settings."""
+
+    class Config(BaseSettingsConfig):
+        """Pydantic Configuration."""
+
+        env_file = ".env"
+        env_file_encoding = core_settings.LOCALE_ENCODING
+
+    BACKENDS: Backends = Backends()
+
+
+backends_settings = BackendSettings()

src/ralph/backends/data/es.py

@@ -18,14 +18,14 @@
     DataBackendStatus,
     enforce_query_checks,
 )
-from ralph.conf import BaseSettingsConfig, CommaSeparatedTuple
+from ralph.conf import BaseSettingsConfig, ClientOptions, CommaSeparatedTuple
 from ralph.exceptions import BackendException, BackendParameterException
 from ralph.utils import parse_bytes_to_dict, read_raw
 
 logger = logging.getLogger(__name__)
 
 
-class ESClientOptions(BaseModel):
+class ESClientOptions(ClientOptions):
     """Elasticsearch additional client options."""
 
     ca_certs: Path = None