Skip to content

Commit

Permalink
Use patch/mock to run tests even when slurm is not installed
Browse files Browse the repository at this point in the history
  • Loading branch information
djperrefort committed Aug 19, 2024
1 parent 07fdef5 commit 3b2bfdb
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 44 deletions.
51 changes: 26 additions & 25 deletions apps/crc_idle.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,29 @@
Resource summaries are provided for GPU and CPU partitions.
"""

from argparse import Namespace
from typing import Dict, Tuple
import re
from argparse import Namespace
from collections import defaultdict

from .utils.cli import BaseParser
from .utils.system_info import Shell, Slurm
from .utils import Shell, Slurm


class CrcIdle(BaseParser):
"""Display idle Slurm resources."""

# The type of resource available on a cluster
# Either ``cores`` or ``GPUs`` depending on the cluster type
cluster_types = {
'smp': 'cores',
'gpu': 'GPUs',
'mpi': 'cores',
'htc': 'cores'
}
default_type = 'cores'
# Either `cores` or `GPUs` depending on the cluster type
cluster_types = defaultdict(
lambda: 'cores',
smp='cores',
gpu='GPUs',
mpi='cores',
htc='cores',
)

def __init__(self) -> None:
"""Define arguments for the command line interface"""
"""Define arguments for the command line interface."""

super(CrcIdle, self).__init__()
self.add_argument('-s', '--smp', action='store_true', help='list idle resources on the smp cluster')
Expand All @@ -37,8 +37,8 @@ def __init__(self) -> None:
self.add_argument('-d', '--htc', action='store_true', help='list idle resources on the htc cluster')
self.add_argument('-p', '--partition', nargs='+', help='only include information for specific partitions')

def get_cluster_list(self, args: Namespace) -> Tuple[str]:
"""Return a list of clusters specified by command line arguments
def get_cluster_list(self, args: Namespace) -> tuple[str]:
"""Return a list of clusters specified by command line arguments.
Returns a tuple of clusters specified by command line arguments. If no
clusters were specified, then return a tuple of all cluster names.
Expand All @@ -50,14 +50,15 @@ def get_cluster_list(self, args: Namespace) -> Tuple[str]:
A tuple of cluster names
"""

argument_options = self.cluster_types
argument_clusters = tuple(filter(lambda cluster: getattr(args, cluster), argument_options))
# Select only the specified clusters
argument_clusters = tuple(self.cluster_types.keys())
specified_clusters = tuple(filter(lambda cluster: getattr(args, cluster), argument_clusters))

# Default to returning all clusters
return argument_clusters or argument_options
return specified_clusters or argument_clusters

@staticmethod
def _idle_cpu_resources(cluster: str, partition: str) -> Dict[int, int]:
def count_idle_cpu_resources(cluster: str, partition: str) -> dict[int, int]:
"""Return the idle CPU resources on a given cluster partition
Args:
Expand All @@ -83,10 +84,10 @@ def _idle_cpu_resources(cluster: str, partition: str) -> Dict[int, int]:
return return_dict

@staticmethod
def _idle_gpu_resources(cluster: str, partition: str) -> Dict[int, int]:
def count_idle_gpu_resources(cluster: str, partition: str) -> dict[int, int]:
"""Return idle GPU resources on a given cluster partition
If the host node is in a ``drain`` state, the GPUs are reported as unavailable.
If the host node is in a `drain` state, the GPUs are reported as unavailable.
Args:
cluster: The cluster to print a summary for
Expand Down Expand Up @@ -122,7 +123,7 @@ def _idle_gpu_resources(cluster: str, partition: str) -> Dict[int, int]:

return return_dict

def count_idle_resources(self, cluster: str, partition: str) -> Dict[int, int]:
def count_idle_resources(self, cluster: str, partition: str) -> dict[int, int]:
"""Determine the number of idle resources on a given cluster partition
The returned dictionary maps the number of idle resources (e.g., cores)
Expand All @@ -136,12 +137,12 @@ def count_idle_resources(self, cluster: str, partition: str) -> Dict[int, int]:
A dictionary mapping idle resources to number of nodes
"""

cluster_type = self.cluster_types.get(cluster, self.default_type)
cluster_type = self.cluster_types[cluster]
if cluster_type == 'GPUs':
return self._idle_gpu_resources(cluster, partition)
return self.count_idle_gpu_resources(cluster, partition)

elif cluster_type == 'cores':
return self._idle_cpu_resources(cluster, partition)
return self.count_idle_cpu_resources(cluster, partition)

raise ValueError(f'Unknown cluster type: {cluster}')

Expand All @@ -157,7 +158,7 @@ def print_partition_summary(self, cluster: str, partition: str) -> None:

output_width = 30
header = f'Cluster: {cluster}, Partition: {partition}'
unit = self.cluster_types.get(cluster, self.default_type)
unit = self.cluster_types[cluster]

print(header)
print('=' * output_width)
Expand Down
2 changes: 2 additions & 0 deletions apps/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
"""The ``utils`` module defines helper utilities for building commandline system tools."""

from .system_info import Shell, Slurm
39 changes: 20 additions & 19 deletions tests/test_crc_idle.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Tests for the ``crc-idle`` application"""
"""Tests for the `crc-idle` application"""

from unittest import TestCase, skip
from argparse import Namespace
from unittest import TestCase
from unittest.mock import patch

from apps.crc_idle import CrcIdle
from apps.utils.system_info import Slurm
Expand Down Expand Up @@ -33,9 +35,9 @@ def test_cluster_parsing(self) -> None:
self.assertFalse(args.htc)
self.assertFalse(args.gpu)

@skip('Requires slurm utilities')
@patch('apps.utils.Slurm.get_cluster_names', new=lambda: tuple(CrcIdle.cluster_types.keys()))
def test_clusters_default_to_false(self) -> None:
"""Test all cluster flags default to a ``False`` value"""
"""Test all cluster flags default to a `False` value"""

app = CrcIdle()
args, unknown_args = app.parse_known_args([])
Expand All @@ -45,25 +47,24 @@ def test_clusters_default_to_false(self) -> None:
self.assertFalse(getattr(args, cluster))


class ClusterList(TestCase):
"""Test the selection of what clusters to print"""
class GetClusterList(TestCase):
"""Test the selection of which clusters to print"""

@skip('Requires slurm utilities')
def test_defaults_all_clusters(self) -> None:
"""Test all clusters are returned if none are specified in the parsed arguments"""
def test_get_cluster_list_no_arguments(self):
"""Test returned values when no clusters are specified."""

app = CrcIdle()
args, unknown_args = app.parse_known_args(['-p', 'partition1'])
self.assertFalse(unknown_args)
args = Namespace(smp=False, gpu=False, mpi=False, invest=False, htc=False, partition=None)
result = app.get_cluster_list(args)

expected = tuple(app.cluster_types.keys())
self.assertEqual(expected, result)

returned_clusters = app.get_cluster_list(args)
self.assertCountEqual(Slurm.get_cluster_names(), returned_clusters)
def test_get_cluster_list_with_cluster_arguments(self):
"""Test returned values when select clusters are specified."""

def test_returns_arg_values(self) -> None:
"""Test returned cluster names match the clusters specified in the parsed arguments"""
app = CrcIdle()
args, unknown_args = app.parse_known_args(['-s', '--mpi'])
self.assertFalse(unknown_args)
args = Namespace(smp=True, gpu=False, mpi=True, invest=False, htc=False, partition=None)
result = app.get_cluster_list(args)

returned_clusters = app.get_cluster_list(args)
self.assertCountEqual(['smp', 'mpi'], returned_clusters)
self.assertEqual(('smp', 'mpi'), result)

0 comments on commit 3b2bfdb

Please sign in to comment.