add math_features and other featuretools

VIDA-NYU · Jun 2, 2024 · 96ce4a1 · 96ce4a1
1 parent 8c3ae48
commit 96ce4a1
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 4 deletions.
diff --git a/alpha_automl/data_profiler.py b/alpha_automl/data_profiler.py
@@ -12,7 +12,7 @@
 
 
 def profile_data(X):
-    metadata = {'nonnumeric_columns': {}, 'useless_columns': [], 'missing_values': False}
+    metadata = {'nonnumeric_columns': {}, 'useless_columns': [], 'missing_values': False, 'numeric_columns': [], 'catagorical_columns': []}
     mapping_encoders = {CATEGORICAL_COLUMN: 'CATEGORICAL_ENCODER', DATETIME_COLUMN: 'DATETIME_ENCODER',
                         TEXT_COLUMN: 'TEXT_ENCODER', IMAGE_COLUMN: 'IMAGE_ENCODER'}
 
@@ -43,6 +43,9 @@ def profile_data(X):
         if 'missing_values_ratio' in profiled_column:
             metadata['missing_values'] = True
 
+    metadata['numeric_columns'] = list(X.select_dtypes(include=['int64', 'float64']).columns)
+    metadata['catagorical_columns'] = list(X.select_dtypes(include=['object', 'category']).columns)
+
     logger.debug(f'Results of profiling data: non-numeric features = {str(metadata["nonnumeric_columns"].keys())}, '
                 f'useless columns = {str(metadata["useless_columns"])}, '
                 f'missing values = {str(metadata["missing_values"])}')

diff --git a/alpha_automl/pipeline_synthesis/pipeline_builder.py b/alpha_automl/pipeline_synthesis/pipeline_builder.py
@@ -8,6 +8,7 @@
 from alpha_automl.utils import create_object, COLUMN_TRANSFORMER_ID, COLUMN_SELECTOR_ID, NATIVE_PRIMITIVE, \
     ADDED_PRIMITIVE
 from alpha_automl.primitive_loader import PRIMITIVE_TYPES
+from feature_engine.creation import MathFeatures
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +38,17 @@ def change_default_hyperparams(primitive_object):
         primitive_object.set_params(algorithm='SAMME')
 
 
+def create_math_features(primitive_type, columns):
+    if primitive_type == "sum":
+        return MathFeatures(variables=columns, func='sum')
+    elif primitive_type == "mean":
+        return MathFeatures(variables=columns, func='mean')
+    elif primitive_type == "std":
+        return MathFeatures(variables=columns, func='std')
+    elif primitive_type == "prod":
+        return MathFeatures(variables=columns, func='prod')
+
+
 def extract_estimators(pipeline_primitives, all_primitives):
     estimators = []
     estimator_name, estimator_obj = pipeline_primitives.pop()
@@ -87,6 +99,7 @@ def make_primitive_objects(self, primitives):
         transformers = []
         nonnumeric_columns = self.metadata['nonnumeric_columns']
         useless_columns = self.metadata['useless_columns']
+        numeric_columns = self.metadata['numeric_columns']
 
         if len(useless_columns) > 0 and len(nonnumeric_columns) == 0:  # Add the transformer to the first step
             selector = (COLUMN_SELECTOR_ID, 'drop', [col_index for col_index, _ in useless_columns])
@@ -105,6 +118,9 @@ def make_primitive_objects(self, primitives):
             elif primitive_type == 'CLASSIFICATION_MULTI_ENSEMBLER' or primitive_type == 'REGRESSION_MULTI_ENSEMBLER':
                 estimators = extract_estimators(pipeline_primitives, self.all_primitives)
                 primitive_object = create_object(primitive_name, {'estimators': estimators})
+            elif "feature_engine.creation" in primitive_name:
+                primitive_name, primitive_name_type = primitive_name.split('-')
+                primitive_object = create_math_features(primitive_name_type, numeric_columns)
             elif self.all_primitives[primitive_name]['origin'] == NATIVE_PRIMITIVE:  # It's an installed primitive
                 primitive_object = create_object(primitive_name, EXTRA_PARAMS.get(primitive_name, None))
             else:

diff --git a/alpha_automl/resource/primitives_hierarchy.json b/alpha_automl/resource/primitives_hierarchy.json
@@ -20,7 +20,11 @@
         "alpha_automl.builtin_primitives.image_encoder.HogTransformer"
     ],
     "FEATURE_GENERATOR": [
-        "sklearn.preprocessing.PolynomialFeatures"
+        "sklearn.preprocessing.PolynomialFeatures",
+        "feature_engine.creation.MathFeatures-sum",
+        "feature_engine.creation.MathFeatures-mean",
+        "feature_engine.creation.MathFeatures-prod",
+        "feature_engine.creation.MathFeatures-std"
     ],
     "FEATURE_SCALER": [
         "sklearn.preprocessing.MaxAbsScaler",
@@ -30,7 +34,9 @@
     "FEATURE_SELECTOR": [
         "sklearn.feature_selection.GenericUnivariateSelect",
         "sklearn.feature_selection.SelectPercentile",
-        "sklearn.feature_selection.SelectKBest"
+        "sklearn.feature_selection.SelectKBest",
+        "feature_engine.selection.SmartCorrelatedSelection",
+        "feature_engine.selection.DropHighPSIFeatures"
     ],
     "COLUMN_TRANSFORMER": [
         "sklearn.compose.ColumnTransformer"

diff --git a/tests/test_data_profiler.py b/tests/test_data_profiler.py
@@ -14,6 +14,8 @@ def test_profile_data():
                                                                  (5, 'country'), (8, 'duration'), (9, 'listed_in'),
                                                                  (10, 'description')],
                                                 'DATETIME_ENCODER': [(6, 'date_added')]},
-                         'useless_columns': [], 'missing_values': True}
+                         'useless_columns': [], 'missing_values': True,
+                         'numeric_columns': ['show_id', 'release_year'],
+                         'catagorical_columns': ['type', 'title', 'director', 'cast', 'country', 'date_added', 'duration', 'listed_in', 'description']}
 
     assert actual_metadata == expected_metadata