diff --git a/climetlab/utils/config.py b/climetlab/utils/config.py index 094c7a74..190bb54d 100644 --- a/climetlab/utils/config.py +++ b/climetlab/utils/config.py @@ -385,6 +385,10 @@ def iter_cubes(self): for loop in self.loops: yield from loop.iterate() + @cached_property + def n_iter_loops(self): + return sum([loop.n_iter_loops for loop in self.loops]) + @property def first_cube(self): return self.first_cube_creator.to_cube() @@ -564,6 +568,10 @@ def repr_lengths(v): lenghts = [f"{k}({repr_lengths(v)})" for k, v in self.values.items()] return f"Loop({self.name}, {','.join(lenghts)}) {self.config}" + @cached_property + def n_iter_loops(self): + return len(list(itertools.product(*self.values.values()))) + def iterate(self): for items in itertools.product(*self.values.values()): yield CubeCreator( @@ -925,10 +933,6 @@ def __init__(self, config, *args, **kwargs): def input_handler(self): return InputHandler(self.loops, self.input, output=self.output) - @cached_property - def n_iter_loops(self): - return sum([loop.n_iter_loops for loop in self.loops]) - def substitute(x, vars=None, ignore_missing=False): """Recursively substitute environment variables and dict values in a nested list ot dict of string. @@ -1073,10 +1077,10 @@ def parse_config(self): self.start = self._config.get("start") if self.start is not None: self.start = to_datetime(self.start) - self.stop = self._config.get("stop") - if self.stop is not None: - self.stop = to_datetime(self.stop) - self.step = self._config.get("step", 1) + self.end = self._config.get("end", self._config.get("stop")) + if self.end is not None: + self.end = to_datetime(self.end) + self.step = self._config.get("step", self._config.get("frequency", 1)) self.group_by = self._config.get("group_by") @@ -1103,7 +1107,7 @@ def __init__(self, config, **kwargs): x = self.start all = [] - while x <= self.stop: + while x <= self.end: all.append(x) x += self.step @@ -1111,8 +1115,8 @@ def __init__(self, config, **kwargs): self.groups = [[format(x) for x in g] for g in result] def parse_config(self): - if "end" in self._config: - raise ValueError(f"Use 'stop' not 'end' in loop. {self._config}") + if "stop" in self._config: + raise ValueError(f"Use 'end' not 'stop' in loop. {self._config}") super().parse_config() def format(self, x): @@ -1132,7 +1136,7 @@ def grouper_key(self, x): def parse_config(self): super().parse_config() assert isinstance(self.start, datetime.date), (type(self.start), self.start) - assert isinstance(self.stop, datetime.date), (type(self.stop), self.stop) + assert isinstance(self.end, datetime.date), (type(self.end), self.end) self.step = datetime.timedelta(days=self.step) def format(self, x): diff --git a/climetlab/version b/climetlab/version index 267d7e01..0cc98846 100644 --- a/climetlab/version +++ b/climetlab/version @@ -1 +1 @@ -0.18.3 +0.18.4 diff --git a/tests/sources/test_url.py b/tests/sources/test_url.py index d46ad00a..5374798e 100644 --- a/tests/sources/test_url.py +++ b/tests/sources/test_url.py @@ -48,7 +48,6 @@ def test_url_ftp_source_anonymous(): @pytest.mark.ftp @pytest.mark.download @pytest.mark.external_download -@pytest.mark.skipif(True, reason="Need to check ftp with password.") def test_url_ftp_source_with_user_pass(): import ftplib