Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adjust "auto" setting for io to process timestamps on default values #730

Merged
merged 3 commits into from
Aug 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions eolearn/core/core_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ def __init__(
:param overwrite_permission: A level of permission for overwriting an existing EOPatch
:param compress_level: A level of data compression and can be specified with an integer from 0 (no compression)
to 9 (highest compression).
:save_timestamps: Whether to save the timestamps of the EOPatch. By default they are saved whenever temporal
features are being saved.
:save_timestamps: Whether to save the timestamps of the EOPatch. With the `"auto"` setting timestamps are saved
if `features=...` or if other temporal features are being saved.
:param use_zarr: Saves numpy-array based features into Zarr files. Requires ZARR extra dependencies.
:param temporal_selection: Writes all of the data to the chosen temporal indices of preexisting arrays. Can be
used for saving data in multiple steps for memory optimization. When set to `"infer"` it will match the
Expand Down Expand Up @@ -185,8 +185,8 @@ def __init__(
default configuration will be taken.
:param features: A collection of features to be loaded. By default, all features will be loaded.
:param lazy_loading: If `True` features will be lazy loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. By default they are loaded whenever temporal
features are being loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. With the `"auto"` setting timestamps are loaded
if `features=...` or if other temporal features are being loaded.
:param temporal_selection: Only loads data corresponding to the chosen indices. Can also be a callable that,
given a list of timestamps, returns a list of booleans declaring which temporal slices to load.
"""
Expand Down
8 changes: 4 additions & 4 deletions eolearn/core/eodata.py
Original file line number Diff line number Diff line change
Expand Up @@ -621,8 +621,8 @@ def save(
to 9 (highest compression).
:param filesystem: An existing filesystem object. If not given it will be initialized according to the `path`
parameter.
:save_timestamps: Whether to save the timestamps of the EOPatch. By default they are saved whenever temporal
features are being saved.
:save_timestamps: Whether to save the timestamps of the EOPatch. With the `"auto"` setting timestamps are saved
if `features=...` or if other temporal features are being saved.
:param use_zarr: Saves numpy-array based features into Zarr files. Requires ZARR extra dependencies.
:param temporal_selection: Writes all of the data to the chosen temporal indices of preexisting arrays. Can be
used for saving data in multiple steps for memory optimization. When set to `"infer"` it will match the
Expand Down Expand Up @@ -661,8 +661,8 @@ def load(
:param lazy_loading: If `True` features will be lazy loaded.
:param filesystem: An existing filesystem object. If not given it will be initialized according to the `path`
parameter.
:load_timestamps: Whether to load the timestamps of the EOPatch. By default they are loaded whenever temporal
features are being loaded.
:load_timestamps: Whether to load the timestamps of the EOPatch. With the `"auto"` setting timestamps are loaded
if `features=...` or if other temporal features are being loaded.
:param temporal_selection: Only loads data corresponding to the chosen indices. Can also be a callable that,
given a list of timestamps, returns a list of booleans declaring which temporal slices to load.
:return: Loaded EOPatch
Expand Down
8 changes: 6 additions & 2 deletions eolearn/core/eodata_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,9 @@ def save_eopatch(
_check_collisions(overwrite_permission, eopatch_features, file_information)

if save_timestamps == "auto":
save_timestamps = any(ftype.is_temporal() for ftype, _ in eopatch_features) and temporal_selection is None
save_timestamps = (
features is ... or any(ftype.is_temporal() for ftype, _ in eopatch_features)
) and temporal_selection is None

# Data must be collected before any tinkering with files due to lazy-loading
data_for_saving = list(
Expand Down Expand Up @@ -306,7 +308,9 @@ def load_eopatch_content(
if file_information.bbox is not None:
bbox = FeatureIOBBox(file_information.bbox, filesystem).load()

auto_load = load_timestamps == "auto" and any(ftype.is_temporal() for ftype, _ in features_dict)
auto_load = load_timestamps == "auto" and (
features is ... or any(ftype.is_temporal() for ftype, _ in features_dict)
)

timestamps = None
if load_timestamps is True or auto_load:
Expand Down
9 changes: 9 additions & 0 deletions tests/core/test_eodata_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,15 @@ def test_save_timestamps(eopatch, fs_loader, save_timestamps, features, should_s
assert temp_fs.exists("/timestamps.json.gz") == should_save


def test_auto_save_load_timestamps(eopatch):
"""Saving and loading with default values should process timestamps."""
test_patch = EOPatch(bbox=eopatch.bbox, timestamps=eopatch.timestamps) # no temporal stuff
with TempFS() as temp_fs:
test_patch.save("/", filesystem=temp_fs)
assert temp_fs.exists("/timestamps.json.gz")
assert EOPatch.load("/", filesystem=temp_fs).timestamps is not None


@mock_s3
@pytest.mark.parametrize("fs_loader", FS_LOADERS)
@pytest.mark.parametrize(
Expand Down