-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #652 from garlic-os/issue-638-do-not-overwrite-netrc
Update RC files instead of overwriting them
- Loading branch information
Showing
8 changed files
with
451 additions
and
206 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| ''' | ||
| When update_rc_file is either True or False, the relevant API RC file should be | ||
| created if it doesn't exist. | ||
| ''' | ||
| from typing import Tuple | ||
|
|
||
| import pytest | ||
| import os | ||
| from pathlib import Path | ||
| from platform import system | ||
| from RAiDER.models import credentials | ||
| from test import random_string | ||
|
|
||
|
|
||
| def get_creds_cds(rc_path: Path) -> Tuple[str, str]: | ||
| import cdsapi | ||
| cds_credentials = cdsapi.api.read_config(rc_path) | ||
| uid, key = cds_credentials['key'].split(':') | ||
| return uid, key | ||
|
|
||
|
|
||
| def get_creds_ecmwf(rc_path: Path) -> Tuple[str, str]: | ||
| import ecmwfapi | ||
| # Get current ECMWF API RC file path | ||
| old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH) | ||
|
|
||
| # Point ecmwfapi to current dir to avoid overwriting ~/.ecmwfapirc | ||
| os.environ["ECMWF_API_RC_FILE"] = str(rc_path) | ||
| key, _, uid = ecmwfapi.api.get_apikey_values() | ||
|
|
||
| # Point ecmwfapi back to previous value and remove local API file | ||
| os.environ["ECMWF_API_RC_FILE"] = old_rc_path | ||
| return uid, key | ||
|
|
||
|
|
||
| def get_creds_netrc(rc_path: Path) -> Tuple[str, str]: | ||
| import netrc | ||
| host = 'urs.earthdata.nasa.gov' | ||
| netrc_credentials = netrc.netrc(rc_path) | ||
| uid, _, key = netrc_credentials.authenticators(host) | ||
| return uid, key | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| 'model_name,get_creds', | ||
| ( | ||
| ('ERA5', get_creds_cds), | ||
| ('ERA5T', get_creds_cds), | ||
| ('HRES', get_creds_ecmwf), | ||
| ('GMAO', get_creds_netrc), | ||
| ('MERRA2', get_creds_netrc) | ||
| ) | ||
| ) | ||
| def test_createFile(model_name, get_creds): | ||
| # Get the rc file's path | ||
| hidden_ext = '_' if system() == "Windows" else '.' | ||
| rc_filename = credentials.RC_FILENAMES[model_name] | ||
| if rc_filename is None: | ||
| return | ||
| rc_path = Path('./') / (hidden_ext + rc_filename) | ||
| rc_path = rc_path.expanduser() | ||
| rc_path.unlink(missing_ok=True) | ||
|
|
||
| test_uid = random_string() | ||
| test_key = random_string() | ||
|
|
||
| # Test creation of the rc file | ||
| credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) | ||
| assert rc_path.exists(), f'{rc_path} does not exist' | ||
|
|
||
| # Check if API is written correctly | ||
| uid, key = get_creds(rc_path) | ||
| rc_path.unlink() | ||
| assert uid == test_uid, f'{rc_path}: UID was not written correctly' | ||
| assert key == test_key, f'{rc_path}: KEY was not written correctly' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| ''' | ||
| Environment variables specific to each model are accepted iff uid and key | ||
| arguments are None. | ||
| ''' | ||
|
|
||
| import pytest | ||
| from pathlib import Path | ||
| from platform import system | ||
| from RAiDER.models import credentials | ||
| from test import random_string | ||
|
|
||
| @pytest.mark.parametrize( | ||
| 'model_name,template,env_var_name_uid,env_var_name_key', | ||
| [ | ||
| ( | ||
| 'ERA5', ( | ||
| 'url: https://cds.climate.copernicus.eu/api/v2\n' | ||
| 'key: {uid}:{key}\n' | ||
| ), | ||
| 'RAIDER_ECMWF_ERA5_UID', | ||
| 'RAIDER_ECMWF_ERA5_API_KEY' | ||
| ), | ||
| ( | ||
| 'ERA5T', ( | ||
| 'url: https://cds.climate.copernicus.eu/api/v2\n' | ||
| 'key: {uid}:{key}\n' | ||
| ), | ||
| 'RAIDER_ECMWF_ERA5_UID', | ||
| 'RAIDER_ECMWF_ERA5_API_KEY' | ||
| ), | ||
| ( | ||
| 'HRES', ( | ||
| '{{\n' | ||
| ' "url" : "https://api.ecmwf.int/v1",\n' | ||
| ' "key" : "{key}",\n' | ||
| ' "email" : "{uid}"\n' | ||
| '}}\n' | ||
| ), | ||
| 'RAIDER_HRES_EMAIL', | ||
| 'RAIDER_HRES_API_KEY' | ||
| ), | ||
| ( | ||
| # Simulate a .netrc file with multiple sets of credentials. | ||
| # The ones not for urs.earthdata.nasa.gov should NOT be touched. | ||
| # Indentation is done with TABS, as that is what the netrc package | ||
| # generates. | ||
| 'GMAO', ( | ||
| 'machine example.com\n' | ||
| ' login johndoe\n' | ||
| ' password hunter2\n' | ||
| 'machine urs.earthdata.nasa.gov\n' | ||
| ' login {uid}\n' | ||
| ' password {key}\n' | ||
| 'machine 127.0.0.1\n' | ||
| ' login bobsmith\n' | ||
| ' password dolphins\n' | ||
| ), | ||
| 'EARTHDATA_USERNAME', | ||
| 'EARTHDATA_PASSWORD' | ||
| ), | ||
| ( | ||
| 'MERRA2', ( | ||
| 'machine example.com\n' | ||
| ' login johndoe\n' | ||
| ' password hunter2\n' | ||
| 'machine urs.earthdata.nasa.gov\n' | ||
| ' login {uid}\n' | ||
| ' password {key}\n' | ||
| 'machine 127.0.0.1\n' | ||
| ' login bobsmith\n' | ||
| ' password dolphins\n' | ||
| ), | ||
| 'EARTHDATA_USERNAME', | ||
| 'EARTHDATA_PASSWORD' | ||
| ), | ||
| ] | ||
| ) | ||
| def test_envVars( | ||
| monkeypatch, | ||
| model_name, | ||
| template, | ||
| env_var_name_uid, | ||
| env_var_name_key | ||
| ): | ||
| hidden_ext = '_' if system() == "Windows" else '.' | ||
| rc_filename = credentials.RC_FILENAMES[model_name] | ||
| if rc_filename is None: | ||
| return | ||
| rc_path = Path('./') / (hidden_ext + rc_filename) | ||
| rc_path = rc_path.expanduser() | ||
| rc_path.unlink(missing_ok=True) | ||
|
|
||
| test_uid = random_string() | ||
| test_key = random_string() | ||
|
|
||
| with monkeypatch.context() as mp: | ||
| mp.setenv(env_var_name_uid, test_uid) | ||
| mp.setenv(env_var_name_key, test_key) | ||
| credentials.check_api(model_name, None, None, './', update_rc_file=False) | ||
|
|
||
| expected_content = template.format(uid=test_uid, key=test_key) | ||
| actual_content = rc_path.read_text() | ||
| rc_path.unlink() | ||
|
|
||
| assert ( | ||
| expected_content == actual_content, | ||
| f'{rc_path} was not updated correctly' | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| ''' | ||
| When update_rc_file is False, the RC file should NOT be modified if it already | ||
| exists. | ||
| ''' | ||
| import pytest | ||
|
|
||
| from pathlib import Path | ||
| from platform import system | ||
| from RAiDER.models import credentials | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('model_name', 'ERA5 ERA5T HRES GMAO MERRA2'.split()) | ||
| def test_updateFalse(model_name): | ||
| # Get the rc file's path | ||
| hidden_ext = '_' if system() == "Windows" else '.' | ||
| rc_filename = credentials.RC_FILENAMES[model_name] | ||
| if rc_filename is None: | ||
| return | ||
| rc_path = Path('./') / (hidden_ext + rc_filename) | ||
| rc_path = rc_path.expanduser() | ||
|
|
||
| # Write some example text to test for | ||
| rc_path.write_text('dummy') | ||
|
|
||
| # Test creation of this model's RC file in current dir | ||
| credentials.check_api(model_name, None, None, './', update_rc_file=False) | ||
|
|
||
| # Assert the content was unchanged | ||
| content = rc_path.read_text() | ||
| rc_path.unlink() | ||
| assert content == 'dummy', f'{rc_path} was modified' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| ''' | ||
| When update_rc_file is True, the RC file should be: | ||
| - updated if it already exists, | ||
| - created if it doesn't, | ||
| - and for .netrc files, it should ONLY update the set of credentials related to | ||
| the given weather model's API URL. | ||
| ''' | ||
| import pytest | ||
|
|
||
| from pathlib import Path | ||
| from platform import system | ||
| from RAiDER.models import credentials | ||
| from test import random_string | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| 'model_name,template', | ||
| [ | ||
| ( | ||
| 'ERA5', ( | ||
| 'url: https://cds.climate.copernicus.eu/api/v2\n' | ||
| 'key: {uid}:{key}\n' | ||
| ) | ||
| ), | ||
| ( | ||
| 'ERA5T', ( | ||
| 'url: https://cds.climate.copernicus.eu/api/v2\n' | ||
| 'key: {uid}:{key}\n' | ||
| ) | ||
| ), | ||
| ( | ||
| 'HRES', ( | ||
| '{{\n' | ||
| ' "url" : "https://api.ecmwf.int/v1",\n' | ||
| ' "key" : "{key}",\n' | ||
| ' "email" : "{uid}"\n' | ||
| '}}\n' | ||
| ) | ||
| ), | ||
| ( | ||
| # Simulate a .netrc file with multiple sets of credentials. | ||
| # The ones not for urs.earthdata.nasa.gov should NOT be touched. | ||
| # Indentation is done with TABS, as that is what the netrc package | ||
| # generates. | ||
| 'GMAO', ( | ||
| 'machine example.com\n' | ||
| ' login johndoe\n' | ||
| ' password hunter2\n' | ||
| 'machine urs.earthdata.nasa.gov\n' | ||
| ' login {uid}\n' | ||
| ' password {key}\n' | ||
| 'machine 127.0.0.1\n' | ||
| ' login bobsmith\n' | ||
| ' password dolphins\n' | ||
| ) | ||
| ), | ||
| ( | ||
| 'MERRA2', ( | ||
| 'machine example.com\n' | ||
| ' login johndoe\n' | ||
| ' password hunter2\n' | ||
| 'machine urs.earthdata.nasa.gov\n' | ||
| ' login {uid}\n' | ||
| ' password {key}\n' | ||
| 'machine 127.0.0.1\n' | ||
| ' login bobsmith\n' | ||
| ' password dolphins\n' | ||
| ) | ||
| ), | ||
| ] | ||
| ) | ||
| def test_updateTrue(model_name, template): | ||
| # Get the rc file's path | ||
| hidden_ext = '_' if system() == "Windows" else '.' | ||
| rc_filename = credentials.RC_FILENAMES[model_name] | ||
| if rc_filename is None: | ||
| return | ||
| rc_path = Path('./') / (hidden_ext + rc_filename) | ||
|
|
||
| test_uid = random_string() | ||
| test_key = random_string() | ||
| credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=True) | ||
|
|
||
| expected_content = template.format(uid=test_uid, key=test_key) | ||
| actual_content = rc_path.read_text() | ||
| rc_path.unlink() | ||
|
|
||
| assert ( | ||
| expected_content == actual_content, | ||
| f'{rc_path} was not updated correctly' | ||
| ) |
Oops, something went wrong.