updated from cds-beta to new cds (#1013)

* updated from cds-beta to new cds

* updated whatsnew

* cleaned up old comment

* url is always read from rcfile, updated tests

dfm_tools/download.py

@@ -98,20 +98,21 @@ def cds_credentials():
     """
     # TODO: put this in a PR at https://github.com/ecmwf/cdsapi
 
-    cds_url = os.environ.get("CDSAPI_URL", "https://cds-beta.climate.copernicus.eu/api")
-    try:
-        # set default for CDSAPI_URL envvar so it does not have to be supplied. This also ignores the URL in ~/.cdsapirc
+    if "CDSAPI_KEY" in os.environ.keys():
+        # in case of envvars, also set CDSAPI_URL envvar so it does not have to be supplied
+        cds_url = os.environ.get("CDSAPI_URL", "https://cds.climate.copernicus.eu/api")
         os.environ["CDSAPI_URL"] = cds_url
+
+    try:
         # gets url/key from env vars or ~/.cdsapirc file
-        # cds_url, cds_apikey, _ = cdsapi.api.get_url_key_verify(url=cds_url, key=None, verify=None)
-        c = cdsapi.Client(url=cds_url)
+        c = cdsapi.Client()
         cds_url = c.url
         cds_apikey = c.key
     except Exception as e:
         if "Missing/incomplete configuration file" in str(e):
             # query apikey if not present in file or envvars
             print("Downloading CDS/ERA5 data requires a ECMWF API-key, copy "
-                  "your API-key from https://cds-beta.climate.copernicus.eu/profile "
+                  "your API-key from https://cds.climate.copernicus.eu/profile "
                   "(first register, login and accept the terms). "
                   "More info in https://forum.ecmwf.int/t/3743.")
             cds_apikey = getpass.getpass("\nEnter your ECMWF API-key (string with dashes): ")
@@ -120,15 +121,18 @@ def cds_credentials():
             raise e
 
     # remove cdsapirc file or env vars if the url/apikey are according to old format
-    if cds_url=="https://cds.climate.copernicus.eu/api/v2":
+    old_urls = [
+        "https://cds.climate.copernicus.eu/api/v2",
+        "https://cds-beta.climate.copernicus.eu/api",
+        ]
+    if cds_url in old_urls:
         # to avoid "HTTPError: 401 Client Error: Unauthorized for url"
         cds_remove_credentials_raise(reason='Old CDS URL found')
     if ":" in cds_apikey:
-        # to avoid "Exception: Not Found" and "HTTPError: 404 Client Error: Not Found for url: https://cds-beta.climate.copernicus.eu/api/resources/dummy"
+        # to avoid "AttributeError: 'Client' object has no attribute 'client'"
         cds_remove_credentials_raise(reason='Old CDS API-key found (with :)')
 
     # check if the authentication works
-    # TODO: requested "check authentication" method in https://github.com/ecmwf/cdsapi/issues/111
     try:
         c = cdsapi.Client()
         c.client.check_authentication()

docs/whats-new.md

@@ -1,5 +1,11 @@
 # What's new
 
+## UNRELEASED
+
+### Fix
+- update from CDS-Beta to new CDS in [#1013](https://github.com/Deltares/dfm_tools/pull/1013)
+
+
 ## 0.28.0 (2024-09-26)
 
 ### Fix

tests/test_download.py

@@ -49,116 +49,152 @@ def test_cds_credentials():
 @pytest.mark.requiressecrets
 @pytest.mark.unittest
 def test_cds_credentials_onlykey_envvars():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
-
-    # remove credentials envvars and file
     with pytest.raises(ValueError):
         cds_remove_credentials_raise()
 
+    # test
     assert "CDSAPI_URL" not in os.environ.keys()
     os.environ["CDSAPI_KEY"] = cds_apikey
-
     cds_credentials()
+
+    # restore credentials file/envvars
     set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.unittest
 def test_cds_credentials_newurl_incorrectkey_rcfile():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
-
-    # remove credentials envvars and file
     with pytest.raises(ValueError):
         cds_remove_credentials_raise()
 
-    cds_url_temp = "https://cds-beta.climate.copernicus.eu/api"
+    # test
+    cds_url_temp = "https://cds.climate.copernicus.eu/api"
     cds_apikey_temp = "INCORRECT-APIKEY"
     cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp)
-
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
     assert "Authentication failed" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)    
+
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.unittest
 def test_cds_credentials_newurl_incorrectkey_envvars():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
+    with pytest.raises(ValueError):
+        cds_remove_credentials_raise()
 
-    os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api"
+    # test
+    os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api"
     os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY"
-
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
     assert "Authentication failed" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
+
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.unittest
 def test_cds_credentials_oldurl_incorrectkey_rcfile():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
-
-    # remove credentials envvars and file
     with pytest.raises(ValueError):
         cds_remove_credentials_raise()
 
+    # test
     cds_url_temp = "https://cds.climate.copernicus.eu/api/v2"
     cds_apikey_temp = "INCORRECT-APIKEY"
     cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp)
+    with pytest.raises(ValueError) as e:
+        cds_credentials()
+    assert "Old CDS URL found" in str(e.value)
+    assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
 
+    # test
+    cds_url_temp = "https://cds-beta.climate.copernicus.eu/api"
+    cds_apikey_temp = "INCORRECT-APIKEY"
+    cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp)
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
-    assert "Authentication failed" in str(e.value) # should actually be "Old CDS URL found", but the url from the file is ignored, which is acceptable
+    assert "Old CDS URL found" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
+
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.unittest
 def test_cds_credentials_oldurl_incorrectkey_envvars():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
+    with pytest.raises(ValueError):
+        cds_remove_credentials_raise()
 
+    # test
     os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api/v2"
     os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY"
+    with pytest.raises(ValueError) as e:
+        cds_credentials()
+    assert "Old CDS URL found" in str(e.value)
+    assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
 
+    # test
+    os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api"
+    os.environ["CDSAPI_KEY"] = "INCORRECT-APIKEY"
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
     assert "Old CDS URL found" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
 
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
+
 
 @pytest.mark.unittest
 def test_cds_credentials_newurl_oldkey_rcfile():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
-
-    # remove credentials envvars and file
     with pytest.raises(ValueError):
         cds_remove_credentials_raise()
 
-    cds_url_temp = "https://cds-beta.climate.copernicus.eu/api"
+    # test
+    cds_url_temp = "https://cds.climate.copernicus.eu/api"
     cds_apikey_temp = "olduid:old-api-key"
     cds_set_credentials_rcfile(cds_url_temp, cds_apikey_temp)
-
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
     assert "Old CDS API-key found (with :)" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
+
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.unittest
 def test_cds_credentials_newurl_oldkey_envvars():
+    # backup credentials and remove credentials envvars and file
     cds_url, cds_apikey = get_cds_url_key()
+    with pytest.raises(ValueError):
+        cds_remove_credentials_raise()
 
-    os.environ["CDSAPI_URL"] = "https://cds-beta.climate.copernicus.eu/api"
+    # test
+    os.environ["CDSAPI_URL"] = "https://cds.climate.copernicus.eu/api"
     os.environ["CDSAPI_KEY"] = "olduid:old-api-key"
-
     with pytest.raises(ValueError) as e:
         cds_credentials()
-    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
     assert "Old CDS API-key found (with :)" in str(e.value)
     assert "The CDS/ECMWF apikey file (~/.cdsapirc) was deleted" in str(e.value)
+
+    # restore credentials file/envvars
+    set_cds_credentials_ifnot_none(cds_url, cds_apikey)
 
 
 @pytest.mark.requiressecrets