From 5ff99e4f2b2a1e8a56bd55a661b14a3f0a645646 Mon Sep 17 00:00:00 2001 From: veenstrajelmer <60435591+veenstrajelmer@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:29:55 +0200 Subject: [PATCH] updated from cds-beta to new cds (#1013) * updated from cds-beta to new cds * updated whatsnew * cleaned up old comment * url is always read from rcfile, updated tests --- dfm_tools/download.py | 22 ++++++----- docs/whats-new.md | 6 +++ tests/test_download.py | 84 ++++++++++++++++++++++++++++++------------ 3 files changed, 79 insertions(+), 33 deletions(-) diff --git a/dfm_tools/download.py b/dfm_tools/download.py index 979f7476..78294c1f 100644 --- a/dfm_tools/download.py +++ b/dfm_tools/download.py @@ -98,20 +98,21 @@ def cds_credentials(): """ # TODO: put this in a PR at https://github.com/ecmwf/cdsapi - cds_url = os.environ.get("CDSAPI_URL", "https://cds-beta.climate.copernicus.eu/api") - try: - # set default for CDSAPI_URL envvar so it does not have to be supplied. This also ignores the URL in ~/.cdsapirc + if "CDSAPI_KEY" in os.environ.keys(): + # in case of envvars, also set CDSAPI_URL envvar so it does not have to be supplied + cds_url = os.environ.get("CDSAPI_URL", "https://cds.climate.copernicus.eu/api") os.environ["CDSAPI_URL"] = cds_url + + try: # gets url/key from env vars or ~/.cdsapirc file - # cds_url, cds_apikey, _ = cdsapi.api.get_url_key_verify(url=cds_url, key=None, verify=None) - c = cdsapi.Client(url=cds_url) + c = cdsapi.Client() cds_url = c.url cds_apikey = c.key except Exception as e: if "Missing/incomplete configuration file" in str(e): # query apikey if not present in file or envvars print("Downloading CDS/ERA5 data requires a ECMWF API-key, copy " - "your API-key from https://cds-beta.climate.copernicus.eu/profile " + "your API-key from https://cds.climate.copernicus.eu/profile " "(first register, login and accept the terms). " "More info in https://forum.ecmwf.int/t/3743.") cds_apikey = getpass.getpass("\nEnter your ECMWF API-key (string with dashes): ") @@ -120,15 +121,18 @@ def cds_credentials(): raise e # remove cdsapirc file or env vars if the url/apikey are according to old format - if cds_url=="https://cds.climate.copernicus.eu/api/v2": + old_urls = [ + "https://cds.climate.copernicus.eu/api/v2", + "https://cds-beta.climate.copernicus.eu/api", + ] + if cds_url in old_urls: # to avoid "HTTPError: 401 Client Error: Unauthorized for url" cds_remove_credentials_raise(reason='Old CDS URL found') if ":" in cds_apikey: - # to avoid "Exception: Not Found" and "HTTPError: 404 Client Error: Not Found for url: https://cds-beta.climate.copernicus.eu/api/resources/dummy" + # to avoid "AttributeError: 'Client' object has no attribute 'client'" cds_remove_credentials_raise(reason='Old CDS API-key found (with :)') # check if the authentication works - # TODO: requested "check authentication" method in https://github.com/ecmwf/cdsapi/issues/111 try: c = cdsapi.Client() c.client.check_authentication() diff --git a/docs/whats-new.md b/docs/whats-new.md index 9fba1eba..b85afa26 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -1,5 +1,11 @@ # What's new +## UNRELEASED + +### Fix +- update from CDS-Beta to new CDS in [#1013](https://github.com/Deltares/dfm_tools/pull/1013) + + ## 0.28.0 (2024-09-26) ### Fix diff --git a/tests/test_download.py b/tests/test_download.py index d2712e96..240e7c25 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -49,116 +49,152 @@ def test_cds_credentials(): @pytest.mark.requiressecrets @pytest.mark.unittest def test_cds_credentials_onlykey_envvars(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() - - # remove credentials envvars and file with pytest.raises(ValueError): cds_remove_credentials_raise() + # test assert "CDSAPI_URL" not in os.environ.keys() os.environ["CDSAPI_KEY"] = cds_apikey - cds_credentials() + + # restore credentials file/envvars set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.unittest def test_cds_credentials_newurl_incorrectkey_rcfile(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() - - # remove credentials envvars and file with pytest.raises(ValueError): cds_remove_credentials_raise() - cds_url_temp = "https://cds-beta.climate.copernicus.eu/api" + # test + cds_url_temp = "https://cds.climate.copernicus.eu/api" cds_apikey_temp = "INCORRECT-APIKEY" cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp) - with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) assert "Authentication failed" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.unittest def test_cds_credentials_newurl_incorrectkey_envvars(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() + with pytest.raises(ValueError): + cds_remove_credentials_raise() - os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api" + # test + os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api" os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY" - with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) assert "Authentication failed" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.unittest def test_cds_credentials_oldurl_incorrectkey_rcfile(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() - - # remove credentials envvars and file with pytest.raises(ValueError): cds_remove_credentials_raise() + # test cds_url_temp = "https://cds.climate.copernicus.eu/api/v2" cds_apikey_temp = "INCORRECT-APIKEY" cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp) + with pytest.raises(ValueError) as e: + cds_credentials() + assert "Old CDS URL found" in str(e.value) + assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + # test + cds_url_temp = "https://cds-beta.climate.copernicus.eu/api" + cds_apikey_temp = "INCORRECT-APIKEY" + cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp) with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) - assert "Authentication failed" in str(e.value) # should actually be "Old CDS URL found", but the url from the file is ignored, which is acceptable + assert "Old CDS URL found" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.unittest def test_cds_credentials_oldurl_incorrectkey_envvars(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() + with pytest.raises(ValueError): + cds_remove_credentials_raise() + # test os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api/v2" os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY" + with pytest.raises(ValueError) as e: + cds_credentials() + assert "Old CDS URL found" in str(e.value) + assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + # test + os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api" + os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY" with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) assert "Old CDS URL found" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) + @pytest.mark.unittest def test_cds_credentials_newurl_oldkey_rcfile(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() - - # remove credentials envvars and file with pytest.raises(ValueError): cds_remove_credentials_raise() - cds_url_temp = "https://cds-beta.climate.copernicus.eu/api" + # test + cds_url_temp = "https://cds.climate.copernicus.eu/api" cds_apikey_temp = "olduid:old-api-key" cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp) - with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) assert "Old CDS API-key found (with :)" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.unittest def test_cds_credentials_newurl_oldkey_envvars(): + # backup credentials and remove credentials envvars and file cds_url, cds_apikey = get_cds_url_key() + with pytest.raises(ValueError): + cds_remove_credentials_raise() - os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api" + # test + os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api" os.environ["CDSAPI_KEY"] = "olduid:old-api-key" - with pytest.raises(ValueError) as e: cds_credentials() - set_cds_credentials_ifnot_none(cds_url, cds_apikey) assert "Old CDS API-key found (with :)" in str(e.value) assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value) + + # restore credentials file/envvars + set_cds_credentials_ifnot_none(cds_url, cds_apikey) @pytest.mark.requiressecrets