Exposing transformations applied by Cleaner and TableVectorizer

CHANGES.rst

@@ -12,6 +12,10 @@ Release 0.10.0
 
 New Features
 ------------
+- The :class:`Cleaner` and :class:`TableVectorizer` classes now have a
+  :method:`list_transformations` method that outputs a human-readable
+  summary of the columns transformed by each of its steps.
+  :pr:`2122` by :user:`Eloi Massoulié <emassoulie>`.
 
 Changes
 -------

skrub/_table_vectorizer.py

@@ -184,6 +184,42 @@ def _get_preprocessors(
     return steps
 
 
+def _list_transformations(estimator):
+    message = ""
+
+    # if isinstance(estimator, TableVectorizer):
+    #     post = estimator._postprocessors
+    # else:
+    #     post = []
+
+    for step in estimator._pipeline.named_steps:
+        if step == "checkinputdataframe":
+            continue
+        transformer = estimator._pipeline.named_steps[step]
+        match transformer.transformer:
+            case DropUninformative():
+                dropped = set(transformer.all_inputs_) - set(transformer.all_outputs_)
+                if dropped:
+                    message += f"DropUninformative ({len(dropped)} columns):" + "\n\t- "
+                    message += "\n\t- ".join(limit_cols(dropped)) + "\n\n"
+            # case ToFloat():
+            #    if transformer not in post:
+            #       message += "Columns transformed to float:" + "\n"
+            #       message += "\n\t".join(transformer.used_inputs_)
+            case ToDatetime():
+                message += (
+                    f"Datetime ({len(transformer.used_inputs_)} columns):" + "\n\t- "
+                )
+                message += "\n\t- ".join(limit_cols(transformer.used_inputs_)) + "\n\n"
+            case CleanNullStrings():
+                message += (
+                    f"Null values cleaned ({len(transformer.used_inputs_)} columns):"
+                    + "\n\t- "
+                )
+                message += "\n\t- ".join(limit_cols(transformer.used_inputs_)) + "\n\n"
+    return message
+
+
 class Cleaner(TransformerMixin, SkrubBaseEstimator):
     """Column-wise consistency checks and sanitization of dtypes, null values and dates.
 
@@ -541,6 +577,9 @@ def get_feature_names_out(self, input_features=None):
         check_is_fitted(self, "all_outputs_")
         return np.asarray(self.all_outputs_)
 
+    def list_transformations(self):
+        return _list_transformations(self)
+
 
 class TableVectorizer(TransformerMixin, SkrubBaseEstimator):
     """Transform a dataframe to a numeric (vectorized) representation.
@@ -1166,3 +1205,60 @@ def get_feature_names_out(self, input_features=None):
         """
         check_is_fitted(self, "all_outputs_")
         return np.asarray(self.all_outputs_)
+
+    def list_transformations(self):
+        """Returns a string reporting the transformations applied by the table
+        vectorizer, and the columns they are each applied to. This covers every
+        preprocessing step, each of the `numeric`, `datetime`, `low cardinality`
+        and `high cardinality` transformations and any specific transformer.
+        """
+        preprocessing_transformations = (
+            "Preprocessors\n=============\n\n" + _list_transformations(self)
+        )
+        vectorize_transformations = "\n\nProcessors by type\n==================\n\n"
+        specific_transformations = ""
+
+        all_transformers = self.kind_to_columns_.copy()
+        specific = all_transformers.pop("specific")
+
+        for transformer_type, transformer_cols in all_transformers.items():
+            if transformer_cols:
+                vectorize_transformations += (
+                    f"{getattr(self, transformer_type).__class__.__name__} "
+                    f"({transformer_type} - {len(transformer_cols)} columns): "
+                    + "\n\t- "
+                )
+                vectorize_transformations += (
+                    "\n\t- ".join(limit_cols(transformer_cols)) + "\n\n"
+                )
+            else:
+                vectorize_transformations += (
+                    f"No {transformer_type} columns have been detected." + "\n\n"
+                )
+
+        if self.specific_transformers:
+            specific_transformations = (
+                "\n\nSpecific transformers\n=====================\n\n"
+            )
+            for t in self.specific_transformers:
+                specific_transformations += (
+                    f"{t[0].__class__.__name__} (specific, {len(specific)} columns):"
+                    + "\n\t- "
+                )
+                specific_transformations += "\n\t- ".join(limit_cols(specific)) + "\n\n"
+
+        return (
+            preprocessing_transformations
+            + "\n\n"
+            + vectorize_transformations
+            + "\n\n"
+            + specific_transformations
+        )
+
+
+def limit_cols(col_names, max_cols=10):
+    if len(col_names) > max_cols:
+        list_cols = list(col_names)[:max_cols] + ["..."]
+    else:
+        list_cols = list(col_names)
+    return list_cols

skrub/tests/test_table_vectorizer.py

@@ -27,6 +27,7 @@
 )
 from skrub._to_float import ToFloat
 from skrub._to_str import ToStr
+from skrub._utils import PassThrough
 from skrub.conftest import _POLARS_INSTALLED
 
 MSG_PANDAS_DEPRECATED_WARNING = "Skip deprecation warning"
@@ -1277,3 +1278,65 @@ def test_duration_to_float(df_module):
     vectorizer = Cleaner()
     transformed = vectorizer.fit_transform(df)
     df_module.assert_column_equal(transformed["duration"], df["duration"])
+
+
+def test_list_transformations(df_module):
+    passthrough_line = [
+        "red",
+        "orange",
+        "yellow",
+        "green",
+        "blue",
+        "indigo",
+        "violet",
+    ]
+    df_dict = {
+        "numbers": [1, 2, 3, 4, 5, 6, None],
+        "low_card": ["up", "up", "up", "down", "down", "up", "down"],
+        "datetime": [
+            "2026-06-01",
+            "2026-06-04",
+            "2026-07-03",
+            "2026-05-29",
+            "2026-01-08",
+            "2026-06-20",
+            None,
+        ],
+        "uninformative": [False, False, False, False, False, False, False],
+    }
+    for i in range(1, 12):
+        df_dict[f"passthrough_{i}"] = passthrough_line
+
+    df = df_module.make_dataframe(df_dict)
+
+    vectorizer = TableVectorizer(
+        specific_transformers=[
+            (PassThrough(), [f"passthrough_{i}" for i in range(1, 12)])
+        ]
+    )
+    _ = vectorizer.fit_transform(df)
+    vectorizer_output = vectorizer.list_transformations()
+    assert vectorizer_output == (
+        "Preprocessors\n=============\n\nNull values cleaned (2 columns):"
+        "\n\t- low_card\n\t- datetime\n\nDatetime (1 columns):\n\t- datetime"
+        "\n\n\n\n\n\nProcessors by type\n==================\n\nPassThrough "
+        "(numeric - 2 columns): \n\t- numbers\n\t- uninformative\n\n"
+        "DatetimeEncoder (datetime - 1 columns): \n\t- datetime\n\nOneHotEncoder"
+        " (low_cardinality - 1 columns): \n\t- low_card\n\nNo high_cardinality"
+        " columns have been detected.\n\n\n\n\n\nSpecific transformers\n"
+        "=====================\n\nPassThrough (specific, 11 columns):\n\t- "
+        "passthrough_1\n\t- passthrough_2\n\t- passthrough_3\n\t- "
+        "passthrough_4\n\t- passthrough_5\n\t- passthrough_6\n\t- passthrough_7"
+        "\n\t- passthrough_8\n\t- passthrough_9\n\t- passthrough_10\n\t- ...\n\n"
+    )
+
+    vectorizer = Cleaner(drop_if_constant=True)
+    _ = vectorizer.fit_transform(df)
+    cleaner_output = vectorizer.list_transformations()
+    assert (
+        cleaner_output == "Null values cleaned (13 columns):\n\t- low_card\n\t"
+        "- datetime\n\t- passthrough_1\n\t- passthrough_2\n\t- passthrough_3\n\t"
+        "- passthrough_4\n\t- passthrough_5\n\t- passthrough_6\n\t- passthrough_7"
+        "\n\t- passthrough_8\n\t- ...\n\nDropUninformative (1 columns):\n\t- "
+        "uninformative\n\nDatetime (1 columns):\n\t- datetime\n\n"
+    )