Merge branch 'ocrd-network-optional-deploy'

ocrd_network/ocrd_network/deployer.py

@@ -17,7 +17,8 @@
 from .deployment_utils import (
     create_docker_client,
     DeployType,
-    wait_for_rabbitmq_availability
+    verify_mongodb_available,
+    verify_rabbitmq_available,
 )
 
 from .runtime_data import (
@@ -229,6 +230,16 @@ def deploy_rabbitmq(
             remove: bool,
             ports_mapping: Union[Dict, None] = None
     ) -> str:
+        if self.data_queue.skip_deployment:
+            self.log.debug(f"RabbitMQ is externaly managed. Skipping deployment")
+            verify_rabbitmq_available(
+                self.data_queue.address,
+                self.data_queue.port,
+                self.data_queue.vhost,
+                self.data_queue.username,
+                self.data_queue.password
+            )
+            return self.data_queue.url
         self.log.debug(f"Trying to deploy '{image}', with modes: "
                        f"detach='{detach}', remove='{remove}'")
 
@@ -271,7 +282,7 @@ def deploy_rabbitmq(
         rmq_port = int(self.data_queue.port)
         rmq_vhost = '/'
 
-        wait_for_rabbitmq_availability(
+        verify_rabbitmq_available(
             host=rmq_host,
             port=rmq_port,
             vhost=rmq_vhost,
@@ -289,6 +300,11 @@ def deploy_mongodb(
             remove: bool,
             ports_mapping: Union[Dict, None] = None
     ) -> str:
+        if self.data_mongo.skip_deployment:
+            self.log.debug('MongoDB is externaly managed. Skipping deployment')
+            verify_mongodb_available(self.data_mongo.url);
+            return self.data_mongo.url
+
         self.log.debug(f"Trying to deploy '{image}', with modes: "
                        f"detach='{detach}', remove='{remove}'")
 
@@ -305,11 +321,20 @@ def deploy_mongodb(
             ports_mapping = {
                 27017: self.data_mongo.port
             }
+        if self.data_mongo.username:
+            environment = [
+                f'MONGO_INITDB_ROOT_USERNAME={self.data_mongo.username}',
+                f'MONGO_INITDB_ROOT_PASSWORD={self.data_mongo.password}'
+            ]
+        else:
+            environment = []
+
         res = client.containers.run(
             image=image,
             detach=detach,
             remove=remove,
-            ports=ports_mapping
+            ports=ports_mapping,
+            environment=environment
         )
         if not res or not res.id:
             raise RuntimeError('Failed to start MongoDB docker container on host: '
@@ -322,7 +347,9 @@ def deploy_mongodb(
         return self.data_mongo.url
 
     def kill_rabbitmq(self) -> None:
-        if not self.data_queue.pid:
+        if self.data_queue.skip_deployment:
+            return
+        elif not self.data_queue.pid:
             self.log.warning('No running RabbitMQ instance found')
             return
         client = create_docker_client(
@@ -337,7 +364,9 @@ def kill_rabbitmq(self) -> None:
         self.log.info('The RabbitMQ is stopped')
 
     def kill_mongodb(self) -> None:
-        if not self.data_mongo.pid:
+        if self.data_mongo.skip_deployment:
+            return
+        elif not self.data_mongo.pid:
             self.log.warning('No running MongoDB instance found')
             return
         client = create_docker_client(

ocrd_network/ocrd_network/deployment_utils.py

@@ -4,8 +4,10 @@
 from docker.transport import SSHHTTPAdapter
 from paramiko import AutoAddPolicy, SSHClient
 from time import sleep
+import re
 
 from .rabbitmq_utils import RMQPublisher
+from pymongo import MongoClient
 
 __all__ = [
     'create_docker_client',
@@ -83,7 +85,7 @@ def _create_paramiko_client(self, base_url: str) -> None:
             self.ssh_client.set_missing_host_key_policy(AutoAddPolicy)
 
 
-def wait_for_rabbitmq_availability(
+def verify_rabbitmq_available(
         host: str,
         port: int,
         vhost: str,
@@ -101,7 +103,16 @@ def wait_for_rabbitmq_availability(
         else:
             # TODO: Disconnect the dummy_publisher here before returning...
             return
-    raise RuntimeError('Error waiting for queue startup: timeout exceeded')
+    raise RuntimeError(f'Cannot connect to RabbitMQ host: {host}, port: {port}, '
+                       f'vhost: {vhost}, username: {username}')
+
+
+def verify_mongodb_available(mongo_url: str) -> None:
+    try:
+        client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0)
+        client.admin.command("ismaster")
+    except Exception:
+        raise RuntimeError(f'Cannot connect to MongoDB: {re.sub(r":[^@]+@", ":****@", mongo_url)}')
 
 
 class DeployType(Enum):

ocrd_network/ocrd_network/runtime_data.py

@@ -98,12 +98,24 @@ class DataMongoDB:
     def __init__(self, config: Dict) -> None:
         self.address = config['address']
         self.port = int(config['port'])
-        self.ssh_username = config['ssh']['username']
-        self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
-        self.ssh_password = config['ssh'].get('password', None)
-        self.username = config['credentials']['username']
-        self.password = config['credentials']['password']
-        self.url = f'mongodb://{self.address}:{self.port}'
+        if 'ssh' in config:
+            self.ssh_username = config['ssh']['username']
+            self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
+            self.ssh_password = config['ssh'].get('password', None)
+        else:
+            self.ssh_username = None
+            self.ssh_keypath = None
+            self.ssh_password = None
+
+        if 'credentials' in config:
+            self.username = config['credentials']['username']
+            self.password = config['credentials']['password']
+            self.url = f'mongodb://{self.username}:{self.password}@{self.address}:{self.port}'
+        else:
+            self.username = None
+            self.password = None
+            self.url = f'mongodb://{self.address}:{self.port}'
+        self.skip_deployment = config.get('skip_deployment', False)
         # Assigned when deployed
         self.pid = None
 
@@ -112,12 +124,19 @@ class DataRabbitMQ:
     def __init__(self, config: Dict) -> None:
         self.address = config['address']
         self.port = int(config['port'])
-        self.ssh_username = config['ssh']['username']
-        self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
-        self.ssh_password = config['ssh'].get('password', None)
+        if 'ssh' in config:
+            self.ssh_username = config['ssh']['username']
+            self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
+            self.ssh_password = config['ssh'].get('password', None)
+        else:
+            self.ssh_username = None
+            self.ssh_keypath = None
+            self.ssh_password = None
+
         self.vhost = '/'
         self.username = config['credentials']['username']
         self.password = config['credentials']['password']
         self.url = f'amqp://{self.username}:{self.password}@{self.address}:{self.port}{self.vhost}'
+        self.skip_deployment = config.get('skip_deployment', False)
         # Assigned when deployed
         self.pid = None

ocrd_validators/ocrd_validators/processing_server_config.schema.yml

@@ -26,6 +26,9 @@ properties:
       ssh:
         description: Information required for an SSH connection
         $ref: "#/$defs/ssh"
+      skip_deployment:
+        description: set to true to deploy queue yourself
+        type: boolean
   database:
     description: Information about the MongoDB
     type: object
@@ -46,6 +49,9 @@ properties:
       ssh:
         description: Information required for an SSH connection
         $ref: "#/$defs/ssh"
+      skip_deployment:
+        description: set to true to deploy database yourself
+        type: boolean
   hosts:
     description: A list of hosts where Processing Servers will be deployed
     type: array