From df3281c0fa6474ac8febe942e5dc23dc82835f3a Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Sun, 11 Feb 2024 16:22:42 -0500 Subject: [PATCH 1/8] Runtime Job Integration --- modules/reana-commons | 1 + modules/reana-db | 1 + reana_workflow_controller/workflow_run_manager.py | 9 +++++++++ 3 files changed, 11 insertions(+) create mode 160000 modules/reana-commons create mode 160000 modules/reana-db diff --git a/modules/reana-commons b/modules/reana-commons new file mode 160000 index 00000000..0798270a --- /dev/null +++ b/modules/reana-commons @@ -0,0 +1 @@ +Subproject commit 0798270afdd265d016a0a6526aa202935c18c976 diff --git a/modules/reana-db b/modules/reana-db new file mode 160000 index 00000000..184d81bd --- /dev/null +++ b/modules/reana-db @@ -0,0 +1 @@ +Subproject commit 184d81bd9144b05ee11d6bfa8cb37e54df1cb4df diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 3e41c2e2..1841387b 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -496,6 +496,7 @@ def _create_job_spec( labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), + "kueue.x-k8s.io/queue-name": "local-queue-batch", }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) @@ -524,6 +525,10 @@ def _create_job_spec( volume_mounts=[], command=["/bin/bash", "-c"], args=command, + # resources=client.V1ResourceRequirements( + # requests={"cpu": "1", "memory": "2Gi"}, + # limits={"cpu": "3", "memory": "4Gi"}, + # ), ) workflow_engine_env_vars.extend( [ @@ -575,6 +580,10 @@ def _create_job_spec( command=["/bin/bash", "-c"], args=self._create_job_controller_startup_cmd(user), ports=[], + # resources=client.V1ResourceRequirements( + # requests={"cpu": "1", "memory": "2Gi"}, + # limits={"cpu": "3", "memory": "4Gi"}, + # ), ) job_controller_env_vars.extend( From 4750a2ea2b2f2283f2d4c4bced2896b43d457cef Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Sat, 17 Feb 2024 10:41:14 -0500 Subject: [PATCH 2/8] Update cluster queue --- reana_workflow_controller/workflow_run_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 1841387b..332db1a5 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -496,7 +496,7 @@ def _create_job_spec( labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), - "kueue.x-k8s.io/queue-name": "local-queue-batch", + "kueue.x-k8s.io/queue-name": "batch-queue-batch", }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) From 9215a1e6eb597fc1c3f55b66c87fc440e83e8173 Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Wed, 10 Apr 2024 10:46:48 -0500 Subject: [PATCH 3/8] Submit Workflow Job with Kueue --- reana_workflow_controller/workflow_run_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 332db1a5..225a3094 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -496,7 +496,7 @@ def _create_job_spec( labels={ "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), - "kueue.x-k8s.io/queue-name": "batch-queue-batch", + "kueue.x-k8s.io/queue-name": "batch-queue-batch", # Submit Workflow Job with Kueue }, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) From d82933a3f914b3a687bf7bbd4f0424f1c3148a4f Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Sun, 14 Apr 2024 22:54:37 -0500 Subject: [PATCH 4/8] feat(kueue): Enable - Disable flags for Kueue --- .../workflow_run_manager.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 225a3094..49e4b129 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -11,7 +11,7 @@ import os from flask import current_app -from kubernetes import client +from kubernetes import client, config from kubernetes.client.models.v1_delete_options import V1DeleteOptions from kubernetes.client.rest import ApiException from reana_commons.config import ( @@ -490,14 +490,23 @@ def _create_job_spec( workspace_mount, workspace_volume = get_workspace_volume( self.workflow.workspace_path ) + + # Set label only if kueue is True + KUEUE_DEPLOYED = True + labels = { + "reana_workflow_mode": "batch", + "reana-run-batch-workflow-uuid": str(self.workflow.id_), + "kueue.x-k8s.io/queue-name": ( + "batch-queue-batch" if KUEUE_DEPLOYED else None + ), + } + + # Filter out None values (labels should not include None) + labels = {k: v for k, v in labels.items() if v is not None} workflow_metadata = client.V1ObjectMeta( name=name, - labels={ - "reana_workflow_mode": "batch", - "reana-run-batch-workflow-uuid": str(self.workflow.id_), - "kueue.x-k8s.io/queue-name": "batch-queue-batch", # Submit Workflow Job with Kueue - }, + labels=labels, namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, ) From ebabd87bc8ddbebe2844500222f673efe57b2637 Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Mon, 15 Apr 2024 15:43:37 -0500 Subject: [PATCH 5/8] feat(kueue): Add variable from Helm --- reana_workflow_controller/config.py | 2 ++ reana_workflow_controller/workflow_run_manager.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py index 1bca65e6..78131b08 100644 --- a/reana_workflow_controller/config.py +++ b/reana_workflow_controller/config.py @@ -172,3 +172,5 @@ RunStatus.pending, ] """Alive workflow statuses.""" + +USE_KUEUE = bool(os.getenv("USE_KUEUE", "false")) \ No newline at end of file diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 49e4b129..f8f8ad2a 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -9,7 +9,6 @@ import json import logging import os - from flask import current_app from kubernetes import client, config from kubernetes.client.models.v1_delete_options import V1DeleteOptions @@ -68,6 +67,7 @@ REANA_WORKFLOW_ENGINE_IMAGE_YADAGE, WORKFLOW_ENGINE_COMMON_ENV_VARS, DEBUG_ENV_VARS, + USE_KUEUE, ) @@ -492,12 +492,20 @@ def _create_job_spec( ) # Set label only if kueue is True - KUEUE_DEPLOYED = True + # KUEUE_DEPLOYED = True + # KUEUE_DEPLOYED = subprocess.run( + # "helm get values kueue -n kueue-system", + # shell=True, + # capture_output=True, + # text=True, + # ) + # KUEUE_DEPLOYED = yaml.safe_load(KUEUE_DEPLOYED.stdout).get("kueueEnabled", False) + labels = { "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), "kueue.x-k8s.io/queue-name": ( - "batch-queue-batch" if KUEUE_DEPLOYED else None + "batch-queue-batch" if USE_KUEUE else None ), } From 57cb646b3a179812e1604453df267e49e8ae02e7 Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Mon, 15 Apr 2024 15:45:30 -0500 Subject: [PATCH 6/8] Clear comments --- reana_workflow_controller/workflow_run_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index f8f8ad2a..9553dc8e 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -491,16 +491,6 @@ def _create_job_spec( self.workflow.workspace_path ) - # Set label only if kueue is True - # KUEUE_DEPLOYED = True - # KUEUE_DEPLOYED = subprocess.run( - # "helm get values kueue -n kueue-system", - # shell=True, - # capture_output=True, - # text=True, - # ) - # KUEUE_DEPLOYED = yaml.safe_load(KUEUE_DEPLOYED.stdout).get("kueueEnabled", False) - labels = { "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), From eb40d9f568fdc7e8c33f9f8e61a3515d18698a05 Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Tue, 23 Apr 2024 14:29:52 -0500 Subject: [PATCH 7/8] feat(kueue): Pass environment variable to job_controller_env_vars --- reana_workflow_controller/config.py | 2 +- reana_workflow_controller/workflow_run_manager.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py index 78131b08..bfc79cec 100644 --- a/reana_workflow_controller/config.py +++ b/reana_workflow_controller/config.py @@ -173,4 +173,4 @@ ] """Alive workflow statuses.""" -USE_KUEUE = bool(os.getenv("USE_KUEUE", "false")) \ No newline at end of file +USE_KUEUE = bool(os.getenv("KUEUE_ENABLED", "False")) \ No newline at end of file diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 9553dc8e..dc844800 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -490,13 +490,11 @@ def _create_job_spec( workspace_mount, workspace_volume = get_workspace_volume( self.workflow.workspace_path ) - + labels = { "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), - "kueue.x-k8s.io/queue-name": ( - "batch-queue-batch" if USE_KUEUE else None - ), + "kueue.x-k8s.io/queue-name": ("batch-queue-batch" if USE_KUEUE else None), } # Filter out None values (labels should not include None) @@ -600,6 +598,7 @@ def _create_job_spec( {"name": "USER", "value": user}, # Required by HTCondor {"name": "K8S_CERN_EOS_AVAILABLE", "value": K8S_CERN_EOS_AVAILABLE}, {"name": "IMAGE_PULL_SECRETS", "value": ",".join(IMAGE_PULL_SECRETS)}, + {"name": "USE_KUEUE", "value": str(USE_KUEUE)}, { "name": "REANA_SQLALCHEMY_DATABASE_URI", "value": SQLALCHEMY_DATABASE_URI, From b927ed2e0fc001e73c7ab42a4abca670979fe616 Mon Sep 17 00:00:00 2001 From: alextintin007 Date: Wed, 8 May 2024 01:18:55 -0500 Subject: [PATCH 8/8] feat(kueue): Fix bug strtobool --- reana_workflow_controller/config.py | 4 ++-- reana_workflow_controller/workflow_run_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py index bfc79cec..58fb9f21 100644 --- a/reana_workflow_controller/config.py +++ b/reana_workflow_controller/config.py @@ -13,7 +13,7 @@ from reana_commons.config import REANA_COMPONENT_PREFIX, SHARED_VOLUME_PATH from reana_db.models import JobStatus, RunStatus - +from distutils.util import strtobool from reana_workflow_controller.version import __version__ SQLALCHEMY_TRACK_MODIFICATIONS = False @@ -173,4 +173,4 @@ ] """Alive workflow statuses.""" -USE_KUEUE = bool(os.getenv("KUEUE_ENABLED", "False")) \ No newline at end of file +USE_KUEUE = bool(strtobool(os.getenv("KUEUE_ENABLED", "False"))) \ No newline at end of file diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index dc844800..ef5473da 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -494,7 +494,7 @@ def _create_job_spec( labels = { "reana_workflow_mode": "batch", "reana-run-batch-workflow-uuid": str(self.workflow.id_), - "kueue.x-k8s.io/queue-name": ("batch-queue-batch" if USE_KUEUE else None), + "kueue.x-k8s.io/queue-name": ("local-queue-batch" if USE_KUEUE else None), } # Filter out None values (labels should not include None)