Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix CMR-related type hints #510

Merged
merged 12 commits into from
Apr 16, 2024
Merged
76 changes: 73 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,78 @@ docs/tutorials/data
tests/integration/data
.ruff_cache

# OS X
notebooks/data/

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Created by https://www.toptal.com/developers/gitignore/api/macos
# Edit at https://www.toptal.com/developers/gitignore?templates=macos

### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride

notebooks/data/
.vscode
# Icon must end with two \r
Icon

# Thumbnails
._*

# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent

# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk

### macOS Patch ###
# iCloud generated files
*.icloud

# End of https://www.toptal.com/developers/gitignore/api/macos

# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode
# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode

### VisualStudioCode ###
.vscode/

# Local History for Visual Studio Code
.history/

# Built Visual Studio Code Extensions
*.vsix

### VisualStudioCode Patch ###
# Ignore all local history of files
.history
.ionide

# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode

# Created by https://www.toptal.com/developers/gitignore/api/direnv
# Edit at https://www.toptal.com/developers/gitignore?templates=direnv

### direnv ###
.direnv
.envrc

# End of https://www.toptal.com/developers/gitignore/api/direnv
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
* fixed 483 by extracting a common CMR query method for collections and granules using SearchAfter header
* Added VCR support for verifying the API call to CMR and the parsing of returned results without relying on CMR availability post development

* Enhancements:
* Corrected and enhanced static type hints for functions and methods that make
CMR queries or handle CMR query results (#508)

## [v0.9.0] 2024-02-28

* Bug fixes:
Expand Down
2 changes: 2 additions & 0 deletions ci/environment-mindeps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ dependencies:
- responses
- pytest
- pytest-cov
- python-magic
- mypy
- types-python-dateutil
- types-requests
- types-setuptools
- ruff
Expand Down
32 changes: 18 additions & 14 deletions earthaccess/api.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
from typing import Any, Dict, List, Optional, Type, Union

import requests
import s3fs
from fsspec import AbstractFileSystem

import earthaccess

from .auth import Auth
from .results import DataGranule
from .results import DataCollection, DataGranule
from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery
from .store import Store
from .typing_ import Any, Dict, List, Optional, Union
from .utils import _validation as validate


Expand All @@ -28,9 +27,7 @@ def _normalize_location(location: Optional[str]) -> Optional[str]:
return location


def search_datasets(
count: int = -1, **kwargs: Any
) -> List[earthaccess.results.DataCollection]:
def search_datasets(count: int = -1, **kwargs: Any) -> List[DataCollection]:
"""Search datasets using NASA's CMR.

[https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
Expand All @@ -54,6 +51,9 @@ def search_datasets(
A list of DataCollection results that can be used to get information about a
dataset, e.g. concept_id, doi, etc.

Raises:
RuntimeError: The CMR query failed.

chuckwondo marked this conversation as resolved.
Show resolved Hide resolved
Examples:
```python
datasets = earthaccess.search_datasets(
Expand All @@ -78,9 +78,7 @@ def search_datasets(
return query.get_all()


def search_data(
count: int = -1, **kwargs: Any
) -> List[earthaccess.results.DataGranule]:
def search_data(count: int = -1, **kwargs: Any) -> List[DataGranule]:
"""Search dataset granules using NASA's CMR.
mfisher87 marked this conversation as resolved.
Show resolved Hide resolved

[https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
Expand All @@ -104,6 +102,9 @@ def search_data(
a list of DataGranules that can be used to access the granule files by using
`download()` or `open()`.

Raises:
RuntimeError: The CMR query failed.

Examples:
```python
datasets = earthaccess.search_data(
Expand Down Expand Up @@ -178,6 +179,9 @@ def download(

Returns:
List of downloaded files

Raises:
Exception: A file download failed.
"""
provider = _normalize_location(provider)
if isinstance(granules, DataGranule):
Expand All @@ -194,7 +198,7 @@ def download(


def open(
granules: Union[List[str], List[earthaccess.results.DataGranule]],
granules: Union[List[str], List[DataGranule]],
provider: Optional[str] = None,
) -> List[AbstractFileSystem]:
"""Returns a list of fsspec file-like objects that can be used to access files
Expand All @@ -216,7 +220,7 @@ def open(
def get_s3_credentials(
daac: Optional[str] = None,
provider: Optional[str] = None,
results: Optional[List[earthaccess.results.DataGranule]] = None,
results: Optional[List[DataGranule]] = None,
) -> Dict[str, Any]:
"""Returns temporary (1 hour) credentials for direct access to NASA S3 buckets. We can
use the daac name, the provider, or a list of results from earthaccess.search_data().
Expand All @@ -239,7 +243,7 @@ def get_s3_credentials(
return earthaccess.__auth__.get_s3_credentials(daac=daac, provider=provider)


def collection_query() -> Type[CollectionQuery]:
def collection_query() -> CollectionQuery:
"""Returns a query builder instance for NASA collections (datasets).

Returns:
Expand All @@ -252,7 +256,7 @@ def collection_query() -> Type[CollectionQuery]:
return query_builder


def granule_query() -> Type[GranuleQuery]:
def granule_query() -> GranuleQuery:
"""Returns a query builder instance for data granules

Returns:
Expand Down Expand Up @@ -311,7 +315,7 @@ def get_requests_https_session() -> requests.Session:
def get_s3fs_session(
daac: Optional[str] = None,
provider: Optional[str] = None,
results: Optional[earthaccess.results.DataGranule] = None,
results: Optional[DataGranule] = None,
) -> s3fs.S3FileSystem:
"""Returns a fsspec s3fs file session for direct access when we are in us-west-2.

Expand Down
Empty file added earthaccess/py.typed
Empty file.
Loading
Loading