From 93e1d010cf616aebe7c369bd1d66456bd3b34bbd Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 16 Apr 2024 16:48:10 +1000 Subject: [PATCH] Handle database URLs a bit better (#1575) * Add url-splitting pseudo option handlers to config. * Update whats_new.rst * Mypyage * Lintage * Fix tests for new behaviour. --- datacube/cfg/opt.py | 56 +++++++++++++++++++++++++++++++--------- docs/about/whats_new.rst | 5 ++++ tests/test_cfg.py | 18 ++++++++----- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/datacube/cfg/opt.py b/datacube/cfg/opt.py index f02499521..4b433b37e 100644 --- a/datacube/cfg/opt.py +++ b/datacube/cfg/opt.py @@ -207,18 +207,50 @@ def validate_and_normalise(self, value: Any) -> Any: def handle_dependent_options(self, value: Any) -> None: if value is None: - for handler in ( - ODCOptionHandler("db_username", self.env, legacy_env_aliases=['DB_USERNAME'], - default=_DEFAULT_DB_USER), - ODCOptionHandler("db_password", self.env, legacy_env_aliases=['DB_PASSWORD']), - ODCOptionHandler("db_hostname", self.env, legacy_env_aliases=['DB_HOSTNAME'], - default=_DEFAULT_HOSTNAME), - IntOptionHandler("db_port", self.env, default=5432, legacy_env_aliases=['DB_PORT'], - minval=1, maxval=49151), - ODCOptionHandler("db_database", self.env, legacy_env_aliases=['DB_DATABASE'], - default=_DEFAULT_DATABASE), - ): - self.env._option_handlers.append(handler) + handlers: tuple[ODCOptionHandler, ...] = ( + ODCOptionHandler("db_username", self.env, legacy_env_aliases=['DB_USERNAME'], + default=_DEFAULT_DB_USER), + ODCOptionHandler("db_password", self.env, legacy_env_aliases=['DB_PASSWORD']), + ODCOptionHandler("db_hostname", self.env, legacy_env_aliases=['DB_HOSTNAME'], + default=_DEFAULT_HOSTNAME), + IntOptionHandler("db_port", self.env, default=5432, legacy_env_aliases=['DB_PORT'], + minval=1, maxval=49151), + ODCOptionHandler("db_database", self.env, legacy_env_aliases=['DB_DATABASE'], + default=_DEFAULT_DATABASE), + ) + else: + # These pseudo-handlers extract the equivalent config from the url returned by this handler. + handlers = ( + PostgresURLPartHandler(self, "username", "db_username", self.env), + PostgresURLPartHandler(self, "password", "db_password", self.env), + PostgresURLPartHandler(self, "hostname", "db_hostname", self.env), + PostgresURLPartHandler(self, "port", "db_port", self.env), + PostgresURLPartHandler(self, "path", "db_database", self.env), + ) + + for handler in handlers: + self.env._option_handlers.append(handler) + + +class PostgresURLPartHandler(ODCOptionHandler): + def __init__(self, urlhandler: PostgresURLOptionHandler, urlpart: str, name: str, env: "ODCEnvironment"): + self.urlhandler = urlhandler + self.urlpart = urlpart + super().__init__(name, env) + + def validate_and_normalise(self, value: Any) -> Any: + url = self.env._normalised[self.urlhandler.name] + purl = urlparse(url) + part = getattr(purl, self.urlpart) + if self.urlpart == "path" and part.startswith('/'): + # Remove leading slash + return str(part)[1:] + else: + return part + + def get_val_from_environment(self) -> str | None: + # Never read from environment - take from URL, wherever it came from + return None def config_options_for_psql_driver(env: "ODCEnvironment"): diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 73c1844e3..be80b6773 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -8,9 +8,14 @@ What's New v1.9.next ========= +v1.9.0-rc4 (15th April 2024) +============================ + - Standardize resampling input supported to `odc.geo.warp.Resampling` (:pull:`1571`) - Refine default behaviour for config engine to support easier migration from 1.8 (:pull:`1573`) - Convert legacy GeoBoxes to odc.geo GeoBoxes in the core API (:pull:`1574`) +- Add URL component pseudo to config layer to expose components to the api when configured as a URL, + and reformat whats_new for 1.9.0-rc4 release. (:pull:`1575`) v1.9.0-rc3 (27th March 2024) diff --git a/tests/test_cfg.py b/tests/test_cfg.py index 34581e604..94807d274 100644 --- a/tests/test_cfg.py +++ b/tests/test_cfg.py @@ -213,8 +213,9 @@ def test_single_env(single_env_config): assert cfg['experimental'].index_driver == "postgis" assert cfg['experimental'].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb" + assert cfg['experimental'].db_username == "foo" with pytest.raises(AttributeError): - assert cfg['experimental'].db_username + assert cfg['experimental'].not_an_option assert cfg['experimental']['db_iam_authentication'] assert cfg['experimental'].db_iam_timeout == 600 assert cfg['experimental']['db_connection_timeout'] == 60 @@ -257,8 +258,9 @@ def assert_simple_options(cfg): assert cfg['default']["db_iam_timeout"] assert cfg['exp2'].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb" + assert cfg['exp2'].db_username == "foo" with pytest.raises(AttributeError): - assert cfg['exp2'].db_username + assert cfg['exp2'].not_an_option assert cfg['exp2']['db_iam_authentication'] assert cfg['exp2'].db_iam_timeout == 300 assert cfg['exp2']['db_connection_timeout'] == 60 @@ -284,8 +286,7 @@ def test_noenv_overrides_in_text(simple_config, monkeypatch): cfg = ODCConfig(text=simple_config) assert cfg["legacy"].db_username != 'bar' - with pytest.raises(AttributeError): - cfg["experimental"].db_username + assert cfg["experimental"].db_username != "bar" @pytest.fixture @@ -389,8 +390,7 @@ def test_envvar_overrides(path_to_yaml_config, monkeypatch): assert cfg["experimental"].db_iam_authentication assert cfg["exp2"].db_iam_authentication assert cfg["exp2"].db_connection_timeout == 20 - with pytest.raises(AttributeError): - assert cfg["experimental"].db_username == 'bar' + assert cfg["experimental"].db_username != 'bar' def test_intopt_validation(): @@ -467,6 +467,12 @@ def test_pgurl_from_config(simple_dict): psql_url_from_config( cfg["memory"] ) + assert cfg["exp2"].db_url == "postgresql://foo:bar@server.subdomain.domain/mytestdb" + assert cfg["exp2"].db_username == "foo" + assert cfg["exp2"].db_password == "bar" + assert cfg["exp2"].db_hostname == "server.subdomain.domain" + assert not cfg["exp2"].db_port + assert cfg["exp2"].db_database == "mytestdb" cfg = ODCConfig(raw_dict={ "foo": {