constant column patch

Author: SamoraHunter

ml_grid/pipeline/data.py

@@ -9,7 +9,7 @@
 from ml_grid.pipeline import read_in
 from ml_grid.pipeline.column_names import get_pertubation_columns
 from ml_grid.pipeline.data_clean_up import clean_up_class
-from ml_grid.pipeline.data_constant_columns import remove_constant_columns
+from ml_grid.pipeline.data_constant_columns import remove_constant_columns, remove_constant_columns_with_debug
 from ml_grid.pipeline.data_correlation_matrix import handle_correlation_matrix
 from ml_grid.pipeline.data_feature_importance_methods import feature_importance_methods
 from ml_grid.pipeline.data_outcome_list import handle_outcome_list
@@ -292,6 +292,14 @@ def __init__(
             self.X_test_orig,
             self.y_test_orig,
         ) = get_data_split(X=self.X, y=self.y, local_param_dict=self.local_param_dict)
+        
+        # Handle columns made constant by splitting
+        self.X_train, self.X_test, self.X_test_orig = remove_constant_columns_with_debug(
+            self.X_train,
+            self.X_test,
+            self.X_test_orig,
+            verbosity=self.verbose
+        )
 
         target_n_features = self.local_param_dict.get("feature_n")
 

ml_grid/pipeline/data_constant_columns.py

@@ -47,4 +47,53 @@ def remove_constant_columns(X: pd.DataFrame, drop_list: Optional[List[str]] = No
         print("Unhandled exception:", str(e))
         raise
 
-    return drop_list
+    return drop_list
+
+def remove_constant_columns_with_debug(X_train, X_test, X_test_orig, verbosity=2):
+    if verbosity > 0:
+        # Debug message: Initial shapes of X_train, X_test, X_test_orig
+        print(f"Initial X_train shape: {X_train.shape}")
+        print(f"Initial X_test shape: {X_test.shape}")
+        print(f"Initial X_test_orig shape: {X_test_orig.shape}")
+    
+    # Calculate the variance for each column in X_train
+    train_variances = X_train.var(axis=0)
+    if verbosity > 1:
+        print(f"Variance of X_train columns:\n{train_variances}")
+    
+    # Identify and remove constant columns in X_train
+    constant_columns_train = train_variances[train_variances == 0].index
+    if verbosity > 0:
+        print(f"Constant columns in X_train: {list(constant_columns_train)}")
+    
+    # Calculate the variance for each column in X_test
+    test_variances = X_test.var(axis=0)
+    if verbosity > 1:
+        print(f"Variance of X_test columns:\n{test_variances}")
+    
+    # Identify constant columns in X_test
+    constant_columns_test = test_variances[test_variances == 0].index
+    if verbosity > 0:
+        print(f"Constant columns in X_test: {list(constant_columns_test)}")
+    
+    # Combine constant columns from both X_train and X_test
+    constant_columns = constant_columns_train.union(constant_columns_test)
+    
+    # Remove the constant columns from both X_train and X_test
+    X_train = X_train.loc[:, ~X_train.columns.isin(constant_columns)]
+    X_test = X_test.loc[:, ~X_test.columns.isin(constant_columns)]
+    
+    # Also remove the same constant columns from X_test_orig
+    X_test_orig = X_test_orig.loc[:, ~X_test_orig.columns.isin(constant_columns)]
+    
+    if verbosity > 0:
+        # Debug message: Shape after removing constant columns from X_train, X_test, X_test_orig
+        print(f"Shape of X_train after removing constant columns: {X_train.shape}")
+        print(f"Shape of X_test after removing constant columns: {X_test.shape}")
+        print(f"Shape of X_test_orig after removing constant columns: {X_test_orig.shape}")
+    
+    # Return the modified X_train, X_test, and X_test_orig, with y_test_orig unchanged
+    return X_train, X_test, X_test_orig
+
+# Example usage with verbosity level 2 (most verbose)
+# X_train, X_test, X_test_orig = remove_constant_columns_with_debug(X_train, X_test, X_test_orig, verbosity=2)