Feat: Upgrade tensorflow to 2.16.1 and keras to 3.4.0

To accommodate the upgrade, next changes to the code were made:

* Since `keras 3.*` now clones the input `regressor` that is provided to the `BaseEstimator` into a `regressor_` attribute, and ONLY updates the `regressor_` one after fitting, metadata extraction in `gordo.builder.build_model.ModelBuilder` was changed to seek the `regressor_` in the `steps`.
* Accessing `keras.optimizers` has changed in `keras 3.*` to having to use the `keras.optimizers.get` method and providing a `class_name` and `config` to deserialize. Relevant for lstm and feedforward autoencoders.
* Model config now requires for `tensorflow.keras.models.Sequential` to define `input_shape` in its layers, otherwise model is not properly built after compiling and prior to fitting. Relevant for `KerasRawModelRegressor`.

`KerasBaseEstimator` underwent the most changes.

* We now need to run the `__init__` of the KerasRegressor with the expected `kwargs` for proper initialisation of the object, but the `kwargs` will always take precedence for `fit`, `predict` and `compile`, so they are mostly for making `keras` happy.
* Saving model for pickling was changed ƒrom `h5` format, which is now considered legacy to `keras` native zip format, and the file is now stored to a temporary file and read into a buffer instead of using the `h5py` library, since `save_model` and `load_model` now exclusively expect an actual file instead of an io buffer.
* Current implementation of model preparation for fit depends on the `__call__` method, which is no longer used in `keras 3.*`, and was changed to `_prepare_model` to replicate the same behaviour right before we call our own `fit` since it requires the `X` and `Y` to be present to set the `n_features` and `n_features_out`.
* `History` is no longer returned from calling `fit` and must be extracted from under `model.history`.
* Manually stored history now resides in `self._history` instead of `self.history` to avoid attribute name clash.

Adjustments to tests were made:

* `input_shape` now resides under `input.shape` in `model.layers`.
* `optimizer_kwargs.lr` is now `optimizer_kwargs.learning_rate`

gordo/builder/build_model.py

@@ -555,13 +555,16 @@ def _extract_metadata_from_model(
         # Continue to look at object values in case, we decided to have a GordoBase
         # which also had a GordoBase as a parameter/attribute, but will satisfy BaseEstimators
         # which can take a GordoBase model as a parameter, which will then have metadata to get
-        for val in model.__dict__.values():
-            if isinstance(val, Pipeline):
-                metadata.update(
-                    ModelBuilder._extract_metadata_from_model(val.steps[-1][1])
-                )
-            elif isinstance(val, GordoBase) or isinstance(val, BaseEstimator):
-                metadata.update(ModelBuilder._extract_metadata_from_model(val))
+        for key, val in model.__dict__.items():
+            if key.endswith(
+                    "_"
+            ):  # keras3 clones the regressor into regressor_ and never updates original regressor
+                if isinstance(val, Pipeline):
+                    metadata.update(
+                        ModelBuilder._extract_metadata_from_model(val.steps[-1][1])
+                    )
+                elif isinstance(val, GordoBase) or isinstance(val, BaseEstimator):
+                    metadata.update(ModelBuilder._extract_metadata_from_model(val))
         return metadata
 
     @property

gordo/machine/model/factories/feedforward_autoencoder.py

@@ -49,7 +49,7 @@ def feedforward_model(
         If str then the name of the optimizer must be provided (e.x. "Adam").
         The arguments of the optimizer can be supplied in optimize_kwargs.
         If a Keras optimizer call the instance of the respective
-        class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
+        class (e.x. Adam(learning_rate=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
         provided Keras default values will be set.
     optimizer_kwargs
         The arguments for the chosen optimizer. If not provided Keras'
@@ -88,8 +88,9 @@ class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
 
     # Instantiate optimizer with kwargs
     if isinstance(optimizer, str):
-        Optim = getattr(keras.optimizers, optimizer)
-        optimizer = Optim(**optimizer_kwargs)
+        optimizer = keras.optimizers.get(
+            {"class_name": optimizer, "config": optimizer_kwargs}
+        )
 
     # Final output layer
     model.add(Dense(n_features_out, activation=out_func))
@@ -132,7 +133,7 @@ def feedforward_symmetric(
         If str then the name of the optimizer must be provided (e.x. "Adam").
         The arguments of the optimizer can be supplied in optimization_kwargs.
         If a Keras optimizer call the instance of the respective
-        class (e.x. ``Adam(lr=0.01,beta_1=0.9, beta_2=0.999)``).  If no arguments are
+        class (e.x. ``Adam(learning_rate=0.01,beta_1=0.9, beta_2=0.999)``).  If no arguments are
         provided Keras default values will be set.
     optimizer_kwargs
         The arguments for the chosen optimizer. If not provided Keras'
@@ -193,7 +194,7 @@ def feedforward_hourglass(
         If str then the name of the optimizer must be provided (e.x. "Adam").
         The arguments of the optimizer can be supplied in optimization_kwargs.
         If a Keras optimizer call the instance of the respective
-        class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
+        class (e.x. Adam(learning_rate=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
         provided Keras default values will be set.
     optimizer_kwargs
         The arguments for the chosen optimizer. If not provided Keras'

gordo/machine/model/factories/lstm_autoencoder.py

@@ -90,8 +90,9 @@ class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)).  If no arguments are
 
     # output layer
     if isinstance(optimizer, str):
-        Optim = getattr(keras.optimizers, optimizer)
-        optimizer = Optim(**optimizer_kwargs)
+        optimizer = keras.optimizers.get(
+            {"class_name": optimizer, "config": optimizer_kwargs}
+        )
 
     model.add(Dense(units=n_features_out, activation=out_func))
 

gordo/machine/model/models.py

@@ -2,25 +2,23 @@
 
 import abc
 import logging
-import io
 import importlib
+import tempfile
 from pprint import pformat
 from typing import Union, Callable, Dict, Any, Optional, Tuple
 from abc import ABCMeta
 from copy import copy, deepcopy
 from importlib.util import find_spec
 
-import h5py
 import tensorflow.keras.models
 from tensorflow.keras.models import load_model, save_model
 from tensorflow.keras.preprocessing.sequence import pad_sequences, TimeseriesGenerator
-from tensorflow.keras.wrappers.scikit_learn import KerasRegressor as BaseWrapper
-from tensorflow.keras.callbacks import History
+from scikeras.wrappers import KerasRegressor
 import numpy as np
 import pandas as pd
 import xarray as xr
 
-from sklearn.base import TransformerMixin, BaseEstimator
+from sklearn.base import TransformerMixin
 from sklearn.metrics import explained_variance_score
 from sklearn.exceptions import NotFittedError
 
@@ -35,7 +33,7 @@
 logger = logging.getLogger(__name__)
 
 
-class KerasBaseEstimator(BaseWrapper, GordoBase, BaseEstimator):
+class KerasBaseEstimator(KerasRegressor, GordoBase):
     supported_fit_args = [
         "batch_size",
         "epochs",
@@ -78,11 +76,15 @@ def __init__(
             building function and/or any additional args to be passed
             to Keras' fit() method
         """
-        self.build_fn = None
-        self.history = None
-
         self.kind = self.load_kind(kind)
         self.kwargs: Dict[str, Any] = kwargs
+        self._history = None
+
+        # This new keras wrapper expects most of these kwargs to be set to the model attributes and uses them for
+        # defaults in some places, but always gives precedence to kwargs passed to respective fit, predict and compile
+        # methods, so this is just to make it happy again
+        _expected_kwargs = {*self._fit_kwargs, *self._predict_kwargs, *self._compile_kwargs}
+        KerasRegressor.__init__(self, **{key: value for key, value in kwargs.items() if key in _expected_kwargs})
 
     @staticmethod
     def parse_module_path(module_path) -> Tuple[Optional[str], str]:
@@ -177,26 +179,26 @@ def __getstate__(self):
 
         state = self.__dict__.copy()
 
-        if hasattr(self, "model") and self.model is not None:
-            buf = io.BytesIO()
-            with h5py.File(buf, compression="lzf", mode="w") as h5:
-                save_model(self.model, h5, overwrite=True, save_format="h5")
-                buf.seek(0)
-                state["model"] = buf
-            if hasattr(self, "history"):
-                from tensorflow.python.keras.callbacks import History
-
-                history = History()
-                history.history = self.history.history
-                history.params = self.history.params
-                history.epoch = self.history.epoch
-                state["history"] = history
+        if self.model is not None:
+            with tempfile.NamedTemporaryFile("w", suffix=".keras") as tf:
+                save_model(self.model, tf.name, overwrite=True)
+                with open(tf.name, "rb") as inf:
+                    state["model"] = inf.read()
+
+            from tensorflow.python.keras.callbacks import History
+
+            history = History()
+            history.history = self._history.history
+            history.params = self._history.params
+            history.epoch = self._history.epoch
+            state["history"] = history
         return state
 
     def __setstate__(self, state):
-        if "model" in state:
-            with h5py.File(state["model"], compression="lzf", mode="r") as h5:
-                state["model"] = load_model(h5, compile=False)
+        if "model" in state and state["model"] is not None:
+            with tempfile.NamedTemporaryFile("wb", suffix=".keras") as tf:
+                tf.write(state["model"])
+                state["model"] = load_model(tf.name, compile=False)
         self.__dict__ = state
         return self
 
@@ -269,9 +271,12 @@ def fit(
         if isinstance(y, (pd.DataFrame, xr.DataArray)):
             y = y.values
         kwargs.setdefault("verbose", 0)
-        history = super().fit(X, y, sample_weight=None, **kwargs)
-        if isinstance(history, History):
-            self.history = history
+
+        if self.model is None:
+            self._prepare_model()
+        model = super().fit(X, y, sample_weight=None, **kwargs)
+        if isinstance(model, KerasRegressor):
+            self._history = model.model.history
         return self
 
     def predict(self, X: np.ndarray, **kwargs) -> np.ndarray:
@@ -301,25 +306,25 @@ def get_params(self, **params):
             Parameters used in this estimator
         """
         params = super().get_params(**params)
-        params.pop("build_fn", None)
+        params.pop("model", None)
         params.update({"kind": self.kind})
         params.update(self.kwargs)
         return params
 
-    def __call__(self):
+    def _prepare_model(self):
         module_name, class_name = self.parse_module_path(self.kind)
         if module_name is None:
             factories = register_model_builder.factories[self.__class__.__name__]
-            build_fn = factories[self.kind]
+            model = factories[self.kind]
         else:
             module = importlib.import_module(module_name)
             if not hasattr(module, class_name):
                 raise ValueError(
                     "kind: %s, unable to find class %s in module '%s'"
                     % (self.kind, class_name, module_name)
                 )
-            build_fn = getattr(module, class_name)
-        return build_fn(**self.sk_params)
+            model = getattr(module, class_name)
+        self.model = model(**self.sk_params)
 
     def get_metadata(self):
         """
@@ -334,9 +339,9 @@ def get_metadata(self):
         -------
             Metadata dictionary, including a history object if present
         """
-        if hasattr(self, "model") and hasattr(self, "history"):
-            history = self.history.history
-            history["params"] = self.history.params
+        if self._history is not None:
+            history = self._history.history
+            history["params"] = self._history.params
             return {"history": history}
         else:
             return {}
@@ -372,7 +377,7 @@ def score(
         -------
             Returns the explained variance score
         """
-        if not hasattr(self, "model"):
+        if self.model is None:
             raise NotFittedError(
                 f"This {self.__class__.__name__} has not been fitted yet."
             )
@@ -404,17 +409,23 @@ class KerasRawModelRegressor(KerasAutoEncoder):
     ...       layers:
     ...         - tensorflow.keras.layers.Dense:
     ...             units: 4
+    ...             input_shape:
+    ...               - 4
     ...         - tensorflow.keras.layers.Dense:
     ...             units: 1
+    ...             input_shape:
+    ...               - 1
     ... '''
     >>> config = yaml.safe_load(config_str)
     >>> model = KerasRawModelRegressor(kind=config)
     >>>
     >>> X, y = np.random.random((10, 4)), np.random.random((10, 1))
     >>> model.fit(X, y, verbose=0)
     KerasRawModelRegressor(kind: {'compile': {'loss': 'mse', 'optimizer': 'adam'},
-     'spec': {'tensorflow.keras.models.Sequential': {'layers': [{'tensorflow.keras.layers.Dense': {'units': 4}},
-                                                                {'tensorflow.keras.layers.Dense': {'units': 1}}]}}})
+     'spec': {'tensorflow.keras.models.Sequential': {'layers': [{'tensorflow.keras.layers.Dense': {'input_shape': [4],
+                                                                                                   'units': 4}},
+                                                                {'tensorflow.keras.layers.Dense': {'input_shape': [1],
+                                                                                                   'units': 1}}]}}})
     >>> out = model.predict(X)
     """
 
@@ -426,7 +437,7 @@ def load_kind(self, kind):
     def __repr__(self):
         return f"{self.__class__.__name__}(kind: {pformat(self.kind)})"
 
-    def __call__(self):
+    def _prepare_model(self):
         """Build Keras model from specification"""
         if not all(k in self.kind for k in self._expected_keys):
             raise ValueError(
@@ -438,9 +449,9 @@ def __call__(self):
 
         # Load any compile kwargs as well, such as compile.optimizer which may map to class obj
         kwargs = serializer.from_definition(self.kind["compile"])
-
         model.compile(**kwargs)
-        return model
+
+        self.model = model
 
 
 class KerasLSTMBaseEstimator(KerasBaseEstimator, TransformerMixin, metaclass=ABCMeta):
@@ -479,7 +490,6 @@ def __init__(
             additional args to be passed to the intermediate fit method.
         """
         self.lookback_window = lookback_window
-        self.batch_size = batch_size
         kwargs["lookback_window"] = lookback_window
         kwargs["kind"] = kind
         kwargs["batch_size"] = batch_size
@@ -541,7 +551,6 @@ def _validate_and_fix_size_of_X(self, X):
     def fit(  # type: ignore
         self, X: np.ndarray, y: np.ndarray, **kwargs
     ) -> "KerasLSTMForecast":
-
         """
         This fits a one step forecast LSTM architecture.
 
@@ -670,7 +679,7 @@ def score(
         -------
             Returns the explained variance score.
         """
-        if not hasattr(self, "model"):
+        if self.model is None:
             raise NotFittedError(
                 f"This {self.__class__.__name__} has not been fitted yet."
             )

gordo/serializer/from_definition.py

@@ -259,7 +259,7 @@ def _build_callbacks(definitions: list):
     --------
     >>> callbacks=_build_callbacks([{'tensorflow.keras.callbacks.EarlyStopping': {'monitor': 'val_loss,', 'patience': 10}}])
     >>> type(callbacks[0])
-    <class 'keras.callbacks.EarlyStopping'>
+    <class 'keras.src.callbacks.early_stopping.EarlyStopping'>
 
     Returns
     -------