Add isort to the lint pipeline (#621)

This commit adds the `isort` library to the StreamFlow lint pipeline, in
order to uniform the way `import` statements are managed and avoid
useless conflicts in GitHub merges.

---------

Co-authored-by: Alberto Mulone <[email protected]>

.github/workflows/ci-tests.yaml

@@ -71,7 +71,7 @@ jobs:
         with:
           node-version: "20"
       - name: "Install Docker (MacOS X)"
-        uses: douglascamata/setup-docker-macos-action@main
+        uses: douglascamata/setup-docker-macos-action@v1-alpha.15
         with:
           colima-additional-options: '--mount /private/var/folders:w --mount-type virtiofs'
         if: ${{ startsWith(matrix.on, 'macos-') }}
@@ -242,7 +242,7 @@ jobs:
         with:
           node-version: "20"
       - name: "Install Docker (MacOs X)"
-        uses: douglascamata/setup-docker-macos-action@main
+        uses: douglascamata/setup-docker-macos-action@v1-alpha.15
         with:
           colima-additional-options: '--mount /private/var/folders:w --mount-type virtiofs'
         if: ${{ startsWith(matrix.on, 'macos-') }}

Makefile

@@ -14,9 +14,11 @@ flake8:
 	flake8 --exclude streamflow/cwl/antlr streamflow tests
 
 format:
+	isort streamflow tests
 	black streamflow tests
 
 format-check:
+	isort --check-only streamflow tests
 	black --diff --check streamflow tests
 
 pyupgrade:

lint-requirements.txt

@@ -1,4 +1,5 @@
 black==24.10.0
 codespell==2.3.0
 flake8-bugbear==24.10.31
+isort==5.13.2
 pyupgrade==3.19.0

pyproject.toml

@@ -115,3 +115,7 @@ omit = [
 
 [tool.black]
 exclude = "streamflow/cwl/antlr"
+
+[tool.isort]
+profile = "black"
+skip = "streamflow/cwl/antlr"

streamflow/config/schema.py

@@ -2,18 +2,13 @@
 
 from streamflow.config import ext_schemas
 from streamflow.core.config import Schema
-from streamflow.cwl.requirement.docker import (
-    cwl_docker_translator_classes,
-)
+from streamflow.cwl.requirement.docker import cwl_docker_translator_classes
 from streamflow.data import data_manager_classes
 from streamflow.deployment import deployment_manager_classes
 from streamflow.deployment.connector import connector_classes
 from streamflow.deployment.filter import binding_filter_classes
 from streamflow.persistence import database_classes
-from streamflow.recovery import (
-    checkpoint_manager_classes,
-    failure_manager_classes,
-)
+from streamflow.recovery import checkpoint_manager_classes, failure_manager_classes
 from streamflow.scheduling import scheduler_classes
 from streamflow.scheduling.policy import policy_classes
 

streamflow/core/config.py

@@ -3,16 +3,16 @@
 import asyncio
 import json
 import posixpath
-from collections.abc import MutableSequence, MutableMapping
-from typing import Any, TYPE_CHECKING, cast
+from collections.abc import MutableMapping, MutableSequence
+from typing import TYPE_CHECKING, Any, cast
 
 from referencing import Registry, Resource
 
 from streamflow.core.exception import WorkflowDefinitionException
 
 if TYPE_CHECKING:
     from streamflow.core.context import SchemaEntity, StreamFlowContext
-    from streamflow.core.deployment import Target, FilterConfig
+    from streamflow.core.deployment import FilterConfig, Target
     from streamflow.core.persistence import DatabaseLoadingContext
 
 

streamflow/core/context.py

@@ -4,7 +4,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import MutableMapping
 from concurrent.futures import ProcessPoolExecutor
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from streamflow.log_handler import logger
 

streamflow/core/data.py

@@ -9,9 +9,10 @@
 from streamflow.core.context import SchemaEntity
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from streamflow.core.context import StreamFlowContext
     from streamflow.core.deployment import ExecutionLocation
-    from typing import Any
 
 
 class DataType(Enum):

streamflow/core/deployment.py

@@ -15,11 +15,12 @@
 from streamflow.core.persistence import DatabaseLoadingContext, PersistableEntity
 
 if TYPE_CHECKING:
-    from streamflow.core.data import StreamWrapperContextManager
+    from typing import Any
+
     from streamflow.core.context import StreamFlowContext
+    from streamflow.core.data import StreamWrapperContextManager
     from streamflow.core.scheduling import AvailableLocation
     from streamflow.core.workflow import Job
-    from typing import Any
 
 
 class ExecutionLocation:

streamflow/core/persistence.py

@@ -4,12 +4,12 @@
 from asyncio import Lock
 from collections.abc import MutableMapping, MutableSequence
 from enum import Enum
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from streamflow.core.context import SchemaEntity, StreamFlowContext
 
 if TYPE_CHECKING:
-    from streamflow.core.deployment import DeploymentConfig, Target, FilterConfig
+    from streamflow.core.deployment import DeploymentConfig, FilterConfig, Target
     from streamflow.core.workflow import Port, Step, Token, Workflow
 
 

streamflow/core/provenance.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
-from collections.abc import MutableSequence, MutableMapping
+from collections.abc import MutableMapping, MutableSequence
 from typing import TYPE_CHECKING
 
 from streamflow.core.persistence import DatabaseLoadingContext

streamflow/core/recovery.py

@@ -7,10 +7,11 @@
 from streamflow.core.context import SchemaEntity
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from streamflow.core.context import StreamFlowContext
     from streamflow.core.data import DataLocation
-    from streamflow.core.workflow import Job, CommandOutput, Step
-    from typing import Any
+    from streamflow.core.workflow import CommandOutput, Job, Step
 
 
 class CheckpointManager(SchemaEntity):

streamflow/core/scheduling.py

@@ -4,10 +4,10 @@
 import os
 from abc import ABC, abstractmethod
 from collections.abc import (
-    MutableSequence,
-    MutableMapping,
-    Iterable,
     Callable,
+    Iterable,
+    MutableMapping,
+    MutableSequence,
     MutableSet,
 )
 from typing import TYPE_CHECKING, cast
@@ -20,9 +20,10 @@
 from streamflow.core.persistence import DatabaseLoadingContext
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from streamflow.core.deployment import Connector, Target
     from streamflow.core.workflow import Job, Status
-    from typing import Any
 
 
 def _check_storages(hardware_a: Hardware, hardware_b: Hardware) -> None:

streamflow/core/utils.py

@@ -9,8 +9,8 @@
 import posixpath
 import shlex
 import uuid
-from collections.abc import MutableSequence, MutableMapping, Iterable
-from typing import Any, TYPE_CHECKING
+from collections.abc import Iterable, MutableMapping, MutableSequence
+from typing import TYPE_CHECKING, Any
 
 from streamflow.core.exception import WorkflowExecutionException
 

streamflow/core/workflow.py

@@ -18,9 +18,10 @@
 )
 
 if TYPE_CHECKING:
-    from streamflow.core.context import StreamFlowContext
     from typing import Any
 
+    from streamflow.core.context import StreamFlowContext
+
 
 class Executor(ABC):
     def __init__(self, workflow: Workflow):

streamflow/cwl/combinator.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import MutableSequence, MutableMapping, AsyncIterable
+from collections.abc import AsyncIterable, MutableMapping, MutableSequence
 from typing import Any, cast
 
 from streamflow.core.context import StreamFlowContext

streamflow/cwl/command.py

@@ -8,10 +8,10 @@
 import shlex
 import time
 from asyncio.subprocess import STDOUT
-from collections.abc import MutableSequence, MutableMapping
+from collections.abc import MutableMapping, MutableSequence
 from decimal import Decimal
 from types import ModuleType
-from typing import Any, IO, cast
+from typing import IO, Any, cast
 
 from ruamel.yaml import RoundTripRepresenter
 from ruamel.yaml.scalarfloat import ScalarFloat
@@ -35,17 +35,8 @@
     WorkflowExecutionException,
 )
 from streamflow.core.persistence import DatabaseLoadingContext
-from streamflow.core.utils import (
-    flatten_list,
-    get_tag,
-)
-from streamflow.core.workflow import (
-    Job,
-    Status,
-    Step,
-    Token,
-    Workflow,
-)
+from streamflow.core.utils import flatten_list, get_tag
+from streamflow.core.workflow import Job, Status, Step, Token, Workflow
 from streamflow.cwl import utils
 from streamflow.cwl.processor import (
     CWLCommandOutput,

streamflow/cwl/hardware.py

@@ -2,8 +2,8 @@
 
 import math
 import os
-from collections.abc import MutableSequence, MutableMapping
-from typing import Any, TYPE_CHECKING
+from collections.abc import MutableMapping, MutableSequence
+from typing import TYPE_CHECKING, Any
 
 from streamflow.core.context import StreamFlowContext
 from streamflow.core.persistence import DatabaseLoadingContext

streamflow/cwl/processor.py

@@ -3,19 +3,15 @@
 import asyncio
 import json
 import logging
-from collections.abc import MutableSequence, MutableMapping, Callable
+from collections.abc import Callable, MutableMapping, MutableSequence
 from typing import Any, cast
 
 import cwl_utils.file_formats
 from schema_salad.exceptions import ValidationException
 
 from streamflow.core.command import CommandOutput, CommandOutputProcessor
 from streamflow.core.context import StreamFlowContext
-from streamflow.core.deployment import (
-    Connector,
-    LocalTarget,
-    Target,
-)
+from streamflow.core.deployment import Connector, LocalTarget, Target
 from streamflow.core.exception import (
     WorkflowDefinitionException,
     WorkflowExecutionException,

streamflow/cwl/requirement/docker/docker.py

@@ -3,7 +3,6 @@
 import os
 import posixpath
 from collections.abc import MutableSequence
-
 from importlib.resources import files
 
 from streamflow.core import utils

streamflow/cwl/requirement/docker/kubernetes.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import tempfile
-
 from importlib.resources import files
+
 from jinja2 import Template
 
 from streamflow.core import utils

streamflow/cwl/requirement/docker/singularity.py

@@ -3,7 +3,6 @@
 import os
 import posixpath
 from collections.abc import MutableSequence
-
 from importlib.resources import files
 
 from streamflow.core import utils

streamflow/cwl/step.py

@@ -5,7 +5,7 @@
 import logging
 import urllib.parse
 from abc import ABC
-from collections.abc import MutableSequence, MutableMapping
+from collections.abc import MutableMapping, MutableSequence
 from typing import Any, cast
 
 from streamflow.core.context import StreamFlowContext
@@ -36,9 +36,9 @@
     ConditionalStep,
     InputInjectorStep,
     LoopOutputStep,
+    ScheduleStep,
     TransferStep,
     _get_token_ids,
-    ScheduleStep,
 )
 from streamflow.workflow.token import IterationTerminationToken, ListToken, ObjectToken
 

streamflow/cwl/transformer.py

@@ -2,7 +2,7 @@
 
 import functools
 import json
-from collections.abc import MutableSequence, MutableMapping
+from collections.abc import MutableMapping, MutableSequence
 from typing import Any, cast
 
 from streamflow.core.context import StreamFlowContext

streamflow/cwl/translator.py

@@ -5,15 +5,11 @@
 import os
 import posixpath
 import urllib.parse
+from collections.abc import MutableMapping, MutableSequence
 from enum import Enum
 from pathlib import PurePosixPath
 from types import ModuleType
-from typing import (
-    Any,
-    cast,
-    get_args,
-)
-from collections.abc import MutableMapping, MutableSequence
+from typing import Any, cast, get_args
 
 import cwl_utils.parser
 import cwl_utils.parser.utils
@@ -34,13 +30,7 @@
     Target,
 )
 from streamflow.core.exception import WorkflowDefinitionException
-from streamflow.core.workflow import (
-    Port,
-    Step,
-    Token,
-    TokenProcessor,
-    Workflow,
-)
+from streamflow.core.workflow import Port, Step, Token, TokenProcessor, Workflow
 from streamflow.cwl import utils
 from streamflow.cwl.combinator import ListMergeCombinator
 from streamflow.cwl.command import (
@@ -79,9 +69,9 @@
 )
 from streamflow.cwl.transformer import (
     AllNonNullTransformer,
-    CWLTokenTransformer,
     CartesianProductSizeTransformer,
     CloneTransformer,
+    CWLTokenTransformer,
     DefaultRetagTransformer,
     DefaultTransformer,
     DotProductSizeTransformer,

streamflow/data/remotepath.py

@@ -15,7 +15,7 @@
 from aiohttp import ClientResponse
 
 from streamflow.core import utils
-from streamflow.core.data import FileType, DataType
+from streamflow.core.data import DataType, FileType
 from streamflow.core.exception import WorkflowExecutionException
 
 if TYPE_CHECKING: