added option num_fill_null=none

Author: dariobrunelli

clearbox_preprocessor/preprocessor.py

@@ -11,18 +11,18 @@
 import warnings
 import numpy as np
 
-from .utils.numerical_transformer import NumericalTransformer
-from .utils.categorical_transformer import CategoricalTransformer
-from .utils.datetime_transformer import DatetimeTransformer
+# from .utils.numerical_transformer import NumericalTransformer
+# from .utils.categorical_transformer import CategoricalTransformer
+# from .utils.datetime_transformer import DatetimeTransformer
 
 # UNCOMMENT FOR DEBUGGING
-# from utils.numerical_transformer import NumericalTransformer
-# from utils.categorical_transformer import CategoricalTransformer
-# from utils.datetime_transformer import DatetimeTransformer
+from utils.numerical_transformer import NumericalTransformer
+from utils.categorical_transformer import CategoricalTransformer
+from utils.datetime_transformer import DatetimeTransformer
 
 class Preprocessor:
     ML_TASKS = {"classification", "regression", None}
-    NUM_FILL_NULL_STRATEGIES = {"interpolate","forward", "backward", "min", "max", "mean", "zero", "one"}
+    NUM_FILL_NULL_STRATEGIES = {"none", "interpolate","forward", "backward", "min", "max", "mean", "zero", "one"}
     SCALING_STRATEGIES = {"none", "normalize", "standardize", "quantile"}
     """
     A class for preprocessing datasets based on polars, including feature selection, handling missing values, scaling, 
@@ -120,7 +120,7 @@ def __init__(
             missing_values_threshold: float = 0.999,
             n_bins: int = 0,
             scaling: Literal["none", "normalize", "standardize", "quantile"] = "none", 
-            num_fill_null : Literal["interpolate","forward", "backward", "min", "max", "mean", "zero", "one"] = "mean",
+            num_fill_null : Literal["none", "interpolate","forward", "backward", "min", "max", "mean", "zero", "one"] = "none",
             unseen_labels = 'ignore',
             ml_task: Literal["classification", "regression", None] = None,
             target_column: str = None,
@@ -662,6 +662,7 @@ def get_categorical_features(self) -> Tuple[str]:
     # real_data = pd.read_csv(os.path.join(file_path,"dataset.csv"))
     # # real_data["income"]      = real_data["income"].map({"<=50K": 0, ">50K": 1})
 
-    preprocessor            = Preprocessor(real_data, num_fill_null=np.nan, scaling='standardize')
+    preprocessor            = Preprocessor(real_data, num_fill_null="none", scaling='quantile')
     real_data_preprocessed  = preprocessor.transform(real_data)
-    df_inverse              = preprocessor.inverse_transform(real_data_preprocessed)
+    df_inverse              = preprocessor.inverse_transform(real_data_preprocessed)
+    pass

clearbox_preprocessor/utils/numerical_transformer.py

@@ -69,16 +69,6 @@ def transform(self, data: pl.DataFrame):
         numerical_features = self.numerical_features
         num_fill_null = self.num_fill_null
 
-        # Fill null values with the specified strategy
-        col_num = pl.col(numerical_features)
-        if isinstance(num_fill_null, str):
-            if num_fill_null == "interpolate":
-                data = data.with_columns(col_num).interpolate()
-            else:
-                data = data.with_columns(col_num.fill_null(strategy=num_fill_null))
-        else:
-            data = data.with_columns(col_num.fill_null(num_fill_null))
-
         # Scale numerical features with the specified method
         match scaling:
             case "none":
@@ -104,6 +94,21 @@ def transform(self, data: pl.DataFrame):
                 # KBinsDiscretizer applied to numerical features
                 data = data.with_columns(numerical_features.qcut(self.n_bins, labels=self.n_bins_labels))
 
+        # Fill null values with the specified strategy
+        col_num = pl.col(numerical_features)
+        if isinstance(num_fill_null, str):
+            if num_fill_null == "interpolate":
+                data = data.with_columns(col_num).interpolate()
+            elif num_fill_null == "none":
+                if scaling in ["quantile", "normalize"]:
+                    data = data.with_columns(col_num.fill_null(-0.01))
+                else:
+                    data = data.with_columns(col_num.fill_null(-10))
+            else:
+                data = data.with_columns(col_num.fill_null(strategy=num_fill_null))
+        else:
+            data = data.with_columns(col_num.fill_null(num_fill_null))
+
         return data
 
 
@@ -132,7 +137,28 @@ def inverse_transform(self, data: pl.DataFrame):
             If an invalid scaling method is provided.
         """
         numerical_features  = self.numerical_features
-        
+        num_fill_null = self.num_fill_null
+        scaling = self.scaling
+
+        # If num_fill_null is "none", convert very negative values to NaN
+        if num_fill_null=="none":
+            if scaling in ["quantile", "normalize"]:
+                for col in numerical_features:
+                    data = data.with_columns(
+                        pl.when(pl.col(col) <= -0.01)
+                        .then(None)
+                        .otherwise(pl.col(col))
+                        .alias(col)
+                    )
+            else:
+                for col in numerical_features:
+                    data = data.with_columns(
+                        pl.when(pl.col(col) <= -10)
+                        .then(None)
+                        .otherwise(pl.col(col))
+                        .alias(col)
+                    )
+
         # Numerical features
         match self.scaling:
             case "none":

setup.py

@@ -11,7 +11,7 @@
 
 setup(
     name="clearbox-preprocessor",  
-    version="0.11.10",  
+    version="0.11.11",  
     author="Dario Brunelli",
     author_email="dario@clearbox.ai",
     description="A fast polars based data pre-processor for ML datasets",