Release version 1.4.2, Merge pull request #672 from sentinel-hub/develop

Release version 1.4.2

.github/workflows/ci_action.yml

@@ -6,9 +6,12 @@ on:
     branches:
       - "master"
       - "develop"
-  schedule:
-    # Schedule events are triggered by whoever last changed the cron schedule
-    - cron: "5 0 * * *"
+  workflow_call:
+
+concurrency:
+  # This will cancel outdated runs on the same pull-request, but not runs for other triggers
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 
 env:
   # The only way to simulate if-else statement
@@ -19,46 +22,45 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ env.CHECKOUT_BRANCH }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
-          architecture: x64
-
-      - name: Prepare pre-commit validators
-        run: |
-          pip install pre-commit
 
-      - name: Check code compliance with pre-commit validators
-        run: pre-commit run --all-files
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files --verbose
 
   check-code-pylint-and-mypy:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ env.CHECKOUT_BRANCH }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
-          architecture: x64
+          # cache: pip # uncomment when all requirements are in `pyproject.toml`
+          # caching the entire environment is faster when cache exists but slower for cache creation
 
       - name: Install packages
         run: |
-          pip install -r requirements-dev.txt --upgrade
+          pip install -r requirements-dev.txt --upgrade --upgrade-strategy eager
           python install_all.py
+          pip install -r ml_tools/requirements-tdigest.txt
 
       - name: Run pylint
         run: make pylint
 
       - name: Run mypy
+        if: success() || failure()
         run: |
           mypy \
             core/eolearn/core \
@@ -77,28 +79,27 @@ jobs:
         python-version:
           - "3.9"
           - "3.10"
+          - "3.11"
         include:
           # A flag marks whether full or partial tests should be run
           # We don't run integration tests on pull requests from outside repos, because they don't have secrets
           - python-version: "3.8"
             full_test_suite: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ env.CHECKOUT_BRANCH }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          # cache: pip # uncomment when all requirements are in `pyproject.toml`
 
       - name: Install packages
         run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential libgdal-dev graphviz proj-bin gcc libproj-dev libspatialindex-dev
-          pip install -r requirements-dev.txt --upgrade
+          pip install -r requirements-dev.txt --upgrade --upgrade-strategy eager
           python install_all.py -e
 
       - name: Run full tests and code coverage
@@ -111,8 +112,7 @@ jobs:
 
       - name: Run reduced tests
         if: ${{ !matrix.full_test_suite }}
-        run: |
-          pytest -m "not sh_integration"
+        run: pytest -m "not sh_integration"
 
       - name: Upload code coverage
         if: ${{ matrix.full_test_suite && github.event_name == 'push' }}

.github/workflows/scheduled_caller.yml

@@ -0,0 +1,11 @@
+name: scheduled build caller
+
+on:
+  schedule:
+    # Schedule events are triggered by whoever last changed the cron schedule
+    - cron: "0 0 * * *"
+
+jobs:
+  call-workflow:
+    uses: sentinel-hub/eo-learn/.github/workflows/ci_action.yml@develop
+    secrets: inherit

.pre-commit-config.yaml

@@ -13,41 +13,18 @@ repos:
       - id: debug-statements
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
         language_version: python3
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.269"
     hooks:
-      - id: isort
-        name: isort (python)
-
-  - repo: https://github.com/PyCQA/autoflake
-    rev: v2.0.1
-    hooks:
-      - id: autoflake
-        args:
-          [
-            --remove-all-unused-imports,
-            --in-place,
-            --ignore-init-module-imports,
-          ]
-
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-bugbear==23.2.13
-          - flake8-comprehensions==3.10.1
-          - flake8-simplify==0.19.3
-          - flake8-typing-imports==1.14.0
+      - id: ruff
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.6.3
+    rev: 1.7.0
     hooks:
       - id: nbqa-black
-      - id: nbqa-isort
-      - id: nbqa-flake8
+      - id: nbqa-ruff

.zenodo.json

@@ -141,6 +141,7 @@
         },
         {
             "name": "Colin Moldenhauer",
+            "affiliation": "Technical University of Munich",
             "type": "Other"
         },
         {

CHANGELOG.md

@@ -1,3 +1,18 @@
+## [Version 1.4.2] - 2023-3-14
+
+- Introduced support for Python 3.11.
+- Removed support for Python 3.7.
+- Added T-Digest `EOTask` in the scope of the Global Earth Monitor Project, contributed by @meengel.
+- Used evalscript generation utility from `sentinelhub-py` in SH related `EOTasks`.
+- Deprecated the `EOPatch.merge` method and extracted it as a function.
+- Deprecated the `OVERWRITE_PATCH` permission and enforcing the usage of explicit string permissions.
+- Encapsulated `FeatureDict` class as `Mapping`, removed inheritance from `dict`.
+- Switched to new-style typed annotations.
+- Introduced the `ruff` python linter, removed `flake8` and `isort` (covered by `ruff`).
+- Fixed issue with occasionally failing scheduled builds on the `master` branch.
+- Various refactoring efforts and dependency improvements.
+- Various improvements to tests and code.
+
 ## [Version 1.4.1] - 2023-3-14
 
 - The codebase is now fully annotated and type annotations are mandatory for all new code.

CONTRIBUTING.md

@@ -131,7 +131,7 @@ This section assumes you have installed all packages in `requirements-dev.txt`.
 
 Most of the automated code-checking is packaged into [pre-commit hooks](https://pre-commit.com/). You can activate them by running `pre-commit install`. If you wish to check all code you can do so by running `pre-commit run --all-files`. This takes care of:
 - auto-formatting the code using `black`, `isort`, and `autoflake`
-- checking the code with `flake8`
+- checking the code with `ruff`
 - checking and formatting any Jupyter notebooks with `nbqa`
 - various other helpful things (correcting line-endings etc.)
 

CREDITS.md

@@ -37,11 +37,12 @@ page or mine the [commit history](https://github.com/sentinel-hub/eo-learn/commi
 ## Other contributors
 
 * Drew Bollinger (DevelopmentSeed)
+* Michael Engel (Technical University of Munich)
 * Peter Fogh
 * Hugo Fournier (Magellium)
 * Ben Huff
 * Filip Koprivec (Jožef Stefan Institute)
-* Colin Moldenhauer
+* Colin Moldenhauer (Technical University of Munich)
 * William Ouellette (TomTom)
 * Radoslav Pitoňák
 * Johannes Schmid (GeoVille)

MANIFEST.in

@@ -1,4 +1,5 @@
 include requirements*.txt
 
 include README.md
+include CREDITS.md
 include LICENSE

README.md

@@ -52,7 +52,7 @@ At the moment there are the following subpackages:
 
 ### PyPi distribution
 
-The package requires Python version **>=3.7** . It can be installed with:
+The package requires Python version **>=3.8** . It can be installed with:
 
 ```bash
 pip install eo-learn
@@ -92,7 +92,7 @@ Some subpackages contain extension modules under `extra` subfolder. Those module
 
 ### Conda Forge distribution
 
-The package requires a Python environment **>=3.7**.
+The package requires a Python environment **>=3.8**.
 
 Thanks to the maintainers of the conda forge feedstock (@benhuff, @dcunn, @mwilson8, @oblute, @rluria14), `eo-learn` can
 be installed using `conda-forge` as follows:
@@ -149,7 +149,7 @@ Examples and introductions to the package can be found [here](https://github.com
 
 ## Contributions
 
-If you would like to contribute to `eo-learn`, check out our [contribution guidelines](./CONTRIBUTING.md).
+The list of all `eo-learn` contributors can be found [here](./CONTRIBUTING.md). If you would like to contribute to `eo-learn`, please check our [contribution guidelines](./CONTRIBUTING.md).
 
 ## Blog posts and papers
 

core/eolearn/core/__init__.py

@@ -22,6 +22,7 @@
     ZipFeatureTask,
 )
 from .eodata import EOPatch
+from .eodata_merge import merge_eopatches
 from .eoexecution import EOExecutor
 from .eonode import EONode, linearly_connect_tasks
 from .eotask import EOTask
@@ -32,4 +33,4 @@
 from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize
 from .utils.parsing import FeatureParser
 
-__version__ = "1.4.1"
+__version__ = "1.4.2"

core/eolearn/core/constants.py

@@ -8,37 +8,38 @@
 """
 import warnings
 from enum import Enum, EnumMeta
-from typing import Any, Optional
+from typing import Any, Optional, TypeVar
 
 from sentinelhub import BBox, MimeType
 from sentinelhub.exceptions import deprecated_function
 
 from .exceptions import EODeprecationWarning
 
 TIMESTAMP_COLUMN = "TIMESTAMP"
+T = TypeVar("T")
 
 
-def _warn_and_adjust(name: str) -> str:
+def _warn_and_adjust(name: T) -> T:
     # since we stick with `UPPER` for attributes and `lower` for values, we include both to reuse function
-    deprecation_msg = None
+    deprecation_msg = None  # placeholder
     if name in ("TIMESTAMP", "timestamp"):
-        name = "TIMESTAMPS" if name == "TIMESTAMP" else "timestamps"
+        name = "TIMESTAMPS" if name == "TIMESTAMP" else "timestamps"  # type: ignore[assignment]
 
     if deprecation_msg:
-        warnings.warn(deprecation_msg, category=EODeprecationWarning, stacklevel=3)  # type: ignore
+        warnings.warn(deprecation_msg, category=EODeprecationWarning, stacklevel=3)  # type: ignore[unreachable]
     return name
 
 
 class EnumWithDeprecations(EnumMeta):
     """A custom EnumMeta class for catching the deprecated Enum members of the FeatureType Enum class."""
 
-    def __getattribute__(cls, name: str) -> Any:
+    def __getattribute__(cls, name: str) -> Any:  # noqa[N805]
         return super().__getattribute__(_warn_and_adjust(name))
 
-    def __getitem__(cls, name: str) -> Any:
+    def __getitem__(cls, name: str) -> Any:  # noqa[N805]
         return super().__getitem__(_warn_and_adjust(name))
 
-    def __call__(cls, value: str, *args: Any, **kwargs: Any) -> Any:
+    def __call__(cls, value: str, *args: Any, **kwargs: Any) -> Any:  # noqa[N805]
         return super().__call__(_warn_and_adjust(value), *args, **kwargs)
 
 
@@ -292,17 +293,54 @@ class FeatureTypeSet(metaclass=DeprecatedCollectionClass):
     RASTER_TYPES_1D = frozenset([FeatureType.SCALAR_TIMELESS, FeatureType.LABEL_TIMELESS])
 
 
-class OverwritePermission(Enum):
-    """Enum class which specifies which content of saved EOPatch can be overwritten when saving new content.
+def _warn_and_adjust_permissions(name: T) -> T:
+    if isinstance(name, str) and name.upper() == "OVERWRITE_PATCH":
+        warnings.warn(
+            '"OVERWRITE_PATCH" permission is deprecated and will be removed in a future version',
+            category=EODeprecationWarning,
+            stacklevel=3,
+        )
+    return name
+
+
+class PermissionsWithDeprecations(EnumMeta):
+    """A custom EnumMeta class for catching the deprecated Enum members of the OverwritePermission Enum class."""
+
+    def __getattribute__(cls, name: str) -> Any:  # noqa[N805]
+        return super().__getattribute__(_warn_and_adjust_permissions(name))
+
+    def __getitem__(cls, name: str) -> Any:  # noqa[N805]
+        return super().__getitem__(_warn_and_adjust_permissions(name))
+
+    def __call__(cls, value: str, *args: Any, **kwargs: Any) -> Any:  # noqa[N805]
+        return super().__call__(_warn_and_adjust_permissions(value), *args, **kwargs)
+
+
+class OverwritePermission(Enum, metaclass=PermissionsWithDeprecations):
+    """Enum class which specifies which content of the saved EOPatch can be overwritten when saving new content.
 
     Permissions are in the following hierarchy:
 
     - `ADD_ONLY` - Only new features can be added, anything that is already saved cannot be changed.
     - `OVERWRITE_FEATURES` - Overwrite only data for features which have to be saved. The remaining content of saved
       EOPatch will stay unchanged.
-    - `OVERWRITE_PATCH` - Overwrite entire content of saved EOPatch and replace it with the new content.
     """
 
-    ADD_ONLY = 0
-    OVERWRITE_FEATURES = 1
-    OVERWRITE_PATCH = 2
+    ADD_ONLY = "ADD_ONLY"
+    OVERWRITE_FEATURES = "OVERWRITE_FEATURES"
+    OVERWRITE_PATCH = "OVERWRITE_PATCH"
+
+    @classmethod
+    def _missing_(cls, value: object) -> "OverwritePermission":
+        permissions_mapping = {0: "ADD_ONLY", 1: "OVERWRITE_FEATURES", 2: "OVERWRITE_PATCH"}
+        if isinstance(value, int) and value in permissions_mapping:
+            deprecation_msg = (
+                f"Please use strings to instantiate overwrite permissions, e.g., instead of {value} use"
+                f" {permissions_mapping[value]!r}"
+            )
+            warnings.warn(deprecation_msg, category=EODeprecationWarning, stacklevel=3)
+
+            return cls(permissions_mapping[value])
+        if isinstance(value, str) and value.upper() in cls._value2member_map_:
+            return cls(value.upper())
+        return super()._missing_(value)