From 1da66f4300677f8b2109666d6d498e46c002549f Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 11:44:16 +0200 Subject: [PATCH 1/5] Fix mongodb credentials usage Previously mongodb credentials from config were accidentially ignored and it was always deployed without credentials --- ocrd_network/ocrd_network/deployer.py | 11 ++++++++++- ocrd_network/ocrd_network/runtime_data.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 491e6a632..f83a19b3e 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -305,11 +305,20 @@ def deploy_mongodb( ports_mapping = { 27017: self.data_mongo.port } + if self.data_mongo.username: + environment = [ + f'MONGO_INITDB_ROOT_USERNAME={self.data_mongo.username}', + f'MONGO_INITDB_ROOT_PASSWORD={self.data_mongo.password}' + ] + else: + environment = [] + res = client.containers.run( image=image, detach=detach, remove=remove, - ports=ports_mapping + ports=ports_mapping, + environment=environment ) if not res or not res.id: raise RuntimeError('Failed to start MongoDB docker container on host: ' diff --git a/ocrd_network/ocrd_network/runtime_data.py b/ocrd_network/ocrd_network/runtime_data.py index 8ab9a2896..254652f1c 100644 --- a/ocrd_network/ocrd_network/runtime_data.py +++ b/ocrd_network/ocrd_network/runtime_data.py @@ -101,9 +101,15 @@ def __init__(self, config: Dict) -> None: self.ssh_username = config['ssh']['username'] self.ssh_keypath = config['ssh'].get('path_to_privkey', None) self.ssh_password = config['ssh'].get('password', None) - self.username = config['credentials']['username'] - self.password = config['credentials']['password'] - self.url = f'mongodb://{self.address}:{self.port}' + + if 'credentials' in config: + self.username = config['credentials']['username'] + self.password = config['credentials']['password'] + self.url = f'mongodb://{self.username}:{self.password}@{self.address}:{self.port}' + else: + self.username = None + self.password = None + self.url = f'mongodb://{self.address}:{self.port}' # Assigned when deployed self.pid = None From 8530bd9006c9bfbf91ef0e9cade582103a2dc04e Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 12:01:47 +0200 Subject: [PATCH 2/5] Add skip_deployment flag for queue and database --- ocrd_network/ocrd_network/runtime_data.py | 25 ++++++++++++++----- .../processing_server_config.schema.yml | 6 +++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/runtime_data.py b/ocrd_network/ocrd_network/runtime_data.py index 254652f1c..59c658ada 100644 --- a/ocrd_network/ocrd_network/runtime_data.py +++ b/ocrd_network/ocrd_network/runtime_data.py @@ -98,9 +98,14 @@ class DataMongoDB: def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) - self.ssh_username = config['ssh']['username'] - self.ssh_keypath = config['ssh'].get('path_to_privkey', None) - self.ssh_password = config['ssh'].get('password', None) + if 'ssh' in config: + self.ssh_username = config['ssh']['username'] + self.ssh_keypath = config['ssh'].get('path_to_privkey', None) + self.ssh_password = config['ssh'].get('password', None) + else: + self.ssh_username = None + self.ssh_keypath = None + self.ssh_password = None if 'credentials' in config: self.username = config['credentials']['username'] @@ -110,6 +115,7 @@ def __init__(self, config: Dict) -> None: self.username = None self.password = None self.url = f'mongodb://{self.address}:{self.port}' + self.skip_deployment = config.get('skip_deployment', False) # Assigned when deployed self.pid = None @@ -118,12 +124,19 @@ class DataRabbitMQ: def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) - self.ssh_username = config['ssh']['username'] - self.ssh_keypath = config['ssh'].get('path_to_privkey', None) - self.ssh_password = config['ssh'].get('password', None) + if 'ssh' in config: + self.ssh_username = config['ssh']['username'] + self.ssh_keypath = config['ssh'].get('path_to_privkey', None) + self.ssh_password = config['ssh'].get('password', None) + else: + self.ssh_username = None + self.ssh_keypath = None + self.ssh_password = None + self.vhost = '/' self.username = config['credentials']['username'] self.password = config['credentials']['password'] self.url = f'amqp://{self.username}:{self.password}@{self.address}:{self.port}{self.vhost}' + self.skip_deployment = config.get('skip_deployment', False) # Assigned when deployed self.pid = None diff --git a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml index 4039e4917..6cbe8d41c 100644 --- a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml +++ b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml @@ -26,6 +26,9 @@ properties: ssh: description: Information required for an SSH connection $ref: "#/$defs/ssh" + skip_deployment: + description: set to true to deploy queue yourself + type: boolean database: description: Information about the MongoDB type: object @@ -46,6 +49,9 @@ properties: ssh: description: Information required for an SSH connection $ref: "#/$defs/ssh" + skip_deployment: + description: set to true to deploy database yourself + type: boolean hosts: description: A list of hosts where Processing Servers will be deployed type: array From b4e65762b2f724642bf590461aa74132231d30f7 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 10 May 2023 12:10:36 +0200 Subject: [PATCH 3/5] Make deployer optionally skip mongo and rabbitmq --- ocrd_network/ocrd_network/deployer.py | 34 +++++++++++++++++-- ocrd_network/ocrd_network/deployment_utils.py | 20 +++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index f83a19b3e..7d67a7e79 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -17,7 +17,9 @@ from .deployment_utils import ( create_docker_client, DeployType, - wait_for_rabbitmq_availability + wait_for_rabbitmq_availability, + verify_mongodb_available, + verify_rabbitmq_available, ) from .runtime_data import ( @@ -229,6 +231,23 @@ def deploy_rabbitmq( remove: bool, ports_mapping: Union[Dict, None] = None ) -> str: + if self.data_queue.skip_deployment: + self.log.debug(f"RabbitMQ is externaly managed. Skipping deployment") + verify_rabbitmq_available( + self.data_queue.address, + self.data_queue.port, + self.data_queue.vhost, + self.data_queue.username, + self.data_queue.password + ) + wait_for_rabbitmq_availability( + host=self.data_queue.address, + port=int(self.data_queue.port), + vhost='/', + username=self.data_queue.username, + password=self.data_queue.password + ) + return self.data_queue.url self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -289,6 +308,11 @@ def deploy_mongodb( remove: bool, ports_mapping: Union[Dict, None] = None ) -> str: + if self.data_mongo.skip_deployment: + self.log.debug('MongoDb is externaly managed. Skipping deployment') + verify_mongodb_available(self.data_mongo.url); + return self.data_mongo.url + self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -331,7 +355,9 @@ def deploy_mongodb( return self.data_mongo.url def kill_rabbitmq(self) -> None: - if not self.data_queue.pid: + if self.data_queue.skip_deployment: + return + elif not self.data_queue.pid: self.log.warning('No running RabbitMQ instance found') return client = create_docker_client( @@ -346,7 +372,9 @@ def kill_rabbitmq(self) -> None: self.log.info('The RabbitMQ is stopped') def kill_mongodb(self) -> None: - if not self.data_mongo.pid: + if self.data_mongo.skip_deployment: + return + elif not self.data_mongo.pid: self.log.warning('No running MongoDB instance found') return client = create_docker_client( diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 9be063cb2..937f2efdb 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -4,8 +4,10 @@ from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient from time import sleep +import re from .rabbitmq_utils import RMQPublisher +from pymongo import MongoClient __all__ = [ 'create_docker_client', @@ -104,6 +106,24 @@ def wait_for_rabbitmq_availability( raise RuntimeError('Error waiting for queue startup: timeout exceeded') +def verify_rabbitmq_available(host: str, port: int, vhost: str, username: str, + password: str) -> None: + try: + dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost) + dummy_publisher.authenticate_and_connect(username=username, password=password) + except Exception: + raise Exception(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + f'vhost: {vhost}, username: {username}') + + +def verify_mongodb_available(mongo_url: str) -> None: + try: + client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) + client.admin.command("ismaster") + except Exception: + raise Exception(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + + class DeployType(Enum): """ Deploy-Type of the processing worker/processor server. """ From 8da78680ae792cf2674d1a29b42780a4e3f5ce1e Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 11 May 2023 09:34:44 +0200 Subject: [PATCH 4/5] Remove redundant rabbitmq availability check --- ocrd_network/ocrd_network/deployer.py | 10 +--------- ocrd_network/ocrd_network/deployment_utils.py | 17 ++++------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 7d67a7e79..b6318d1a8 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -17,7 +17,6 @@ from .deployment_utils import ( create_docker_client, DeployType, - wait_for_rabbitmq_availability, verify_mongodb_available, verify_rabbitmq_available, ) @@ -240,13 +239,6 @@ def deploy_rabbitmq( self.data_queue.username, self.data_queue.password ) - wait_for_rabbitmq_availability( - host=self.data_queue.address, - port=int(self.data_queue.port), - vhost='/', - username=self.data_queue.username, - password=self.data_queue.password - ) return self.data_queue.url self.log.debug(f"Trying to deploy '{image}', with modes: " f"detach='{detach}', remove='{remove}'") @@ -290,7 +282,7 @@ def deploy_rabbitmq( rmq_port = int(self.data_queue.port) rmq_vhost = '/' - wait_for_rabbitmq_availability( + verify_rabbitmq_available( host=rmq_host, port=rmq_port, vhost=rmq_vhost, diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 937f2efdb..b02d5d277 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -85,7 +85,7 @@ def _create_paramiko_client(self, base_url: str) -> None: self.ssh_client.set_missing_host_key_policy(AutoAddPolicy) -def wait_for_rabbitmq_availability( +def verify_rabbitmq_available( host: str, port: int, vhost: str, @@ -103,17 +103,8 @@ def wait_for_rabbitmq_availability( else: # TODO: Disconnect the dummy_publisher here before returning... return - raise RuntimeError('Error waiting for queue startup: timeout exceeded') - - -def verify_rabbitmq_available(host: str, port: int, vhost: str, username: str, - password: str) -> None: - try: - dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost) - dummy_publisher.authenticate_and_connect(username=username, password=password) - except Exception: - raise Exception(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' - f'vhost: {vhost}, username: {username}') + raise RuntimeError(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + f'vhost: {vhost}, username: {username}') def verify_mongodb_available(mongo_url: str) -> None: @@ -121,7 +112,7 @@ def verify_mongodb_available(mongo_url: str) -> None: client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) client.admin.command("ismaster") except Exception: - raise Exception(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + raise RuntimeError(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') class DeployType(Enum): From f749c2426abfc39f4f980e3f896a78d4da1960dd Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 15 May 2023 14:40:29 +0200 Subject: [PATCH 5/5] Correct spelling --- ocrd_network/ocrd_network/deployer.py | 2 +- ocrd_network/ocrd_network/deployment_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index b6318d1a8..f5b7f045d 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -301,7 +301,7 @@ def deploy_mongodb( ports_mapping: Union[Dict, None] = None ) -> str: if self.data_mongo.skip_deployment: - self.log.debug('MongoDb is externaly managed. Skipping deployment') + self.log.debug('MongoDB is externaly managed. Skipping deployment') verify_mongodb_available(self.data_mongo.url); return self.data_mongo.url diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index b02d5d277..a5c01de6e 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -103,7 +103,7 @@ def verify_rabbitmq_available( else: # TODO: Disconnect the dummy_publisher here before returning... return - raise RuntimeError(f'Cannot connet to Rabbitmq host: {host}, port: {port}, ' + raise RuntimeError(f'Cannot connect to RabbitMQ host: {host}, port: {port}, ' f'vhost: {vhost}, username: {username}') @@ -112,7 +112,7 @@ def verify_mongodb_available(mongo_url: str) -> None: client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0) client.admin.command("ismaster") except Exception: - raise RuntimeError(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}') + raise RuntimeError(f'Cannot connect to MongoDB: {re.sub(r":[^@]+@", ":****@", mongo_url)}') class DeployType(Enum):