diff --git a/docs/user_guide/creation/RelativeFeatures.rst b/docs/user_guide/creation/RelativeFeatures.rst index 5611aa204..870b5c1cd 100644 --- a/docs/user_guide/creation/RelativeFeatures.rst +++ b/docs/user_guide/creation/RelativeFeatures.rst @@ -141,6 +141,51 @@ Which will return the names of all the variables in the transformed data: 'Marks_pow_Age'] +With polars +----------- + +:class:`RelativeFeatures()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.creation import RelativeFeatures + + df = pl.DataFrame({ + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + }) + + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age"], + func = ["sub", "div", "mod", "pow"], + ) + + print(transformer.fit_transform(df)) + +The resulting values match those found with pandas (`Age_pow_Age`'s large +values are genuine `int64` overflow from raising `Age` to the power of +itself, not an error - the same happens with pandas): + +.. code:: text + + shape: (4, 10) + ┌─────┬───────┬─────────────┬───────────────┬─────────────┬───────────────┬─────────────┬───────────────┬──────────────────────┬───────────────┐ + │ Age ┆ Marks ┆ Age_sub_Age ┆ Marks_sub_Age ┆ Age_div_Age ┆ Marks_div_Age ┆ Age_mod_Age ┆ Marks_mod_Age ┆ Age_pow_Age ┆ Marks_pow_Age │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ i64 ┆ f64 ┆ f64 ┆ f64 ┆ i64 ┆ f64 ┆ i64 ┆ f64 │ + ╞═════╪═══════╪═════════════╪═══════════════╪═════════════╪═══════════════╪═════════════╪═══════════════╪══════════════════════╪═══════════════╡ + │ 20 ┆ 0.9 ┆ 0 ┆ -19.1 ┆ 1.0 ┆ 0.045 ┆ 0 ┆ 0.9 ┆ -2101438300051996672 ┆ 0.121577 │ + │ 21 ┆ 0.8 ┆ 0 ┆ -20.2 ┆ 1.0 ┆ 0.038095 ┆ 0 ┆ 0.8 ┆ -1595931050845505211 ┆ 0.009223 │ + │ 19 ┆ 0.7 ┆ 0 ┆ -18.3 ┆ 1.0 ┆ 0.036842 ┆ 0 ┆ 0.7 ┆ 6353754964178307979 ┆ 0.00114 │ + │ 18 ┆ 0.6 ┆ 0 ┆ -17.4 ┆ 1.0 ┆ 0.033333 ┆ 0 ┆ 0.6 ┆ -497033925936021504 ┆ 0.000102 │ + └─────┴───────┴─────────────┴───────────────┴─────────────┴───────────────┴─────────────┴───────────────┴──────────────────────┴───────────────┘ + +`fill_value`, `drop_original`, and `get_feature_names_out()` work +identically to the pandas examples above. + + Additional resources -------------------- diff --git a/feature_engine/creation/relative_features.py b/feature_engine/creation/relative_features.py index 5b2957bff..d664c2448 100644 --- a/feature_engine/creation/relative_features.py +++ b/feature_engine/creation/relative_features.py @@ -1,6 +1,8 @@ from typing import List, Union -import pandas as pd +import narwhals as nw +import numpy as np +from narwhals.typing import IntoDataFrame from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, @@ -31,6 +33,20 @@ "pow", ] +_NUMPY_OPS = { + "add": np.add, + "sub": np.subtract, + "mul": np.multiply, + "div": np.divide, + "truediv": np.true_divide, + "floordiv": np.floor_divide, + "mod": np.mod, + "pow": np.power, +} + +# these can divide by zero; fill_value handling applies only to them. +_DIVISION_LIKE = {"div", "truediv", "floordiv", "mod"} + @Substitution( variables=_variables_numerical_docstring, @@ -54,12 +70,10 @@ class RelativeFeatures(BaseCreation): features to / by a group of reference variables. The features resulting from these functions are added to the dataframe. - This transformer works only with numerical variables. It uses the pandas methods - `pd.DataFrame.add`, `pd.DataFrame.sub`, `pd.DataFrame.mul`, `pd.DataFrame.div`, - `pd.DataFrame.truediv`, `pd.DataFrame.floordiv`, `pd.DataFrame.mod` and - `pd.DataFrame.pow`. - Find out more in `pandas documentation - `_. + This transformer works only with numerical variables. It uses NumPy's `add`, + `subtract`, `multiply`, `divide`, `true_divide`, `floor_divide`, `mod` and + `power` under the hood, matching the semantics of the equivalent pandas + `DataFrame.add`, `DataFrame.sub`, etc. methods. More details in the :ref:`User Guide `. @@ -125,6 +139,27 @@ class RelativeFeatures(BaseCreation): 0 1 4 3 0.333333 1.333333 1 2 5 4 0.500000 1.250000 2 3 6 5 0.600000 1.200000 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import RelativeFeatures + >>> X = pl.DataFrame({"x1": [1, 2, 3], "x2": [4, 5, 6], "x3": [3, 4, 5]}) + >>> rf = RelativeFeatures(variables=["x1", "x2"], + >>> reference=["x3"], + >>> func=["div"]) + >>> rf.fit(X) + >>> rf.transform(X) + shape: (3, 5) + ┌─────┬─────┬─────┬───────────┬───────────┐ + │ x1 ┆ x2 ┆ x3 ┆ x1_div_x3 ┆ x2_div_x3 │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ i64 ┆ i64 ┆ f64 ┆ f64 │ + ╞═════╪═════╪═════╪═══════════╪═══════════╡ + │ 1 ┆ 4 ┆ 3 ┆ 0.333333 ┆ 1.333333 │ + │ 2 ┆ 5 ┆ 4 ┆ 0.5 ┆ 1.25 │ + │ 3 ┆ 6 ┆ 5 ┆ 0.6 ┆ 1.2 │ + └─────┴─────┴─────┴───────────┴───────────┘ """ def __init__( @@ -179,124 +214,70 @@ def __init__( self.func = func self.fill_value = fill_value - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Add new features. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe + X_new: dataframe The input dataframe plus the new variables. """ X = self._check_transform_input_and_state(X) - methods_dict = { - "add": self._add, - "mul": self._mul, - "sub": self._sub, - "div": self._div, - "truediv": self._truediv, - "floordiv": self._floordiv, - "mod": self._mod, - "pow": self._pow, - } + nw_X = nw.from_native(X, eager_only=True) + # Extract each column as its own 1D array (not one batched 2D array + # via select().to_numpy()) so mixed int/float variables each keep + # their own dtype promotion, matching pandas' per-column .sub()/ + # .div()/etc. instead of upcasting everything to a common dtype. + var_arrays = {var: nw_X.get_column(var).to_numpy() for var in self.variables} + ref_arrays = {ref: nw_X.get_column(ref).to_numpy() for ref in self.reference} + new_series = [] for func in self.func: - methods_dict[func](X) - - if self.drop_original: - X.drop( - columns=set(self.variables + self.reference), - inplace=True, - ) - - return X - - def _sub(self, X): - for reference in self.reference: - varname = [f"{var}_sub_{reference}" for var in self.variables] - X[varname] = X[self.variables].sub(X[reference], axis=0) - return X - - def _add(self, X): - for reference in self.reference: - varname = [f"{var}_add_{reference}" for var in self.variables] - X[varname] = X[self.variables].add(X[reference], axis=0) - return X - - def _mul(self, X): - for reference in self.reference: - varname = [f"{var}_mul_{reference}" for var in self.variables] - X[varname] = X[self.variables].mul(X[reference], axis=0) - return X - - def _div(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_div_{reference}" for var in self.variables] - X[varname] = X[self.variables].div(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _truediv(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_truediv_{reference}" for var in self.variables] - X[varname] = X[self.variables].truediv(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _floordiv(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_floordiv_{reference}" for var in self.variables] - X[varname] = X[self.variables].floordiv(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _mod(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_mod_{reference}" for var in self.variables] - X[varname] = X[self.variables].mod(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _pow(self, X): - for reference in self.reference: - varname = [f"{var}_pow_{reference}" for var in self.variables] - X[varname] = X[self.variables].pow(X[reference], axis=0) - return X + op = _NUMPY_OPS[func] + for reference in self.reference: + ref_arr = ref_arrays[reference] + + if func in _DIVISION_LIKE: + zero_mask = ref_arr == 0 + contains_zero = zero_mask.any() + if self.fill_value is None and contains_zero: + self._raise_error_when_zero_in_denominator() + + for var in self.variables: + name = f"{var}_{func}_{reference}" + if func in _DIVISION_LIKE: + with np.errstate(divide="ignore", invalid="ignore"): + result = op(var_arrays[var], ref_arr) + if contains_zero: + # floordiv/mod on integer input stay integer-typed; + # widen to match fill_value if it wouldn't fit, + # mirroring pandas' automatic dtype promotion. + fill_arr = np.asarray(self.fill_value) + if not np.can_cast(fill_arr, result.dtype, casting="safe"): + result = result.astype( + np.result_type(result.dtype, fill_arr.dtype) + ) + result[zero_mask] = self.fill_value + else: + result = op(var_arrays[var], ref_arr) + + new_series.append( + nw.new_series(name, result, backend=nw_X.implementation) + ) + + nw_X = nw_X.with_columns(*new_series) + if self.drop_original is True: + nw_X = nw_X.drop(list(set(self.variables + self.reference))) + + return nw_X.to_native() def _raise_error_when_zero_in_denominator(self): raise ValueError( @@ -305,11 +286,6 @@ def _raise_error_when_zero_in_denominator(self): "or set `fill_value` to a number." ) - def _find_zeroes_in_reference(self, X, var): - zero_ix = X[var] == 0 - zero_bool = (zero_ix).any() - return zero_ix, zero_bool - def _get_new_features_name(self) -> List: """Return names of the created features.""" diff --git a/tests/test_creation/test_relative_features.py b/tests/test_creation/test_relative_features.py index dbfa4972c..e8dc5971c 100644 --- a/tests/test_creation/test_relative_features.py +++ b/tests/test_creation/test_relative_features.py @@ -1,10 +1,34 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.pipeline import Pipeline from feature_engine.creation import RelativeFeatures +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + + +def _none_to_nan(values): + # Missing values print as None for polars, NaN for pandas float columns + # - both mean "missing" here, so normalize both sides before comparing. + return [np.nan if v is None else v for v in values] + + +def assert_df_equal(X, expected: dict, abs_tol: float = 1e-5) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert _none_to_nan(result[col]) == pytest.approx( + _none_to_nan(values), abs=abs_tol, nan_ok=True + ) + def test_mandatory_init_parameters(): with pytest.raises(TypeError): @@ -75,14 +99,16 @@ def test_error_when_drop_original_not_bool(): ) -def test_error_when_variables_not_numeric(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_variables_not_numeric(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Name", "Age", "Marks"], reference=["Age", "Name"], func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) transformer = RelativeFeatures( reference=["Name", "Age", "Marks"], @@ -90,17 +116,19 @@ def test_error_when_variables_not_numeric(df_vartypes): func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) -def test_error_when_entered_variables_not_in_df(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_entered_variables_not_in_df(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["FeatOutsideDataset", "Age"], reference=["Age", "Name"], func=["sub"], ) with pytest.raises(KeyError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) transformer = RelativeFeatures( reference=["FeatOutsideDataset", "Age"], @@ -108,146 +136,126 @@ def test_error_when_entered_variables_not_in_df(df_vartypes): func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) - + transformer.fit_transform(df) -def test_classic_binary_operation(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_classic_binary_operation(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=["sub", "div", "add", "mul"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Age_div_Marks"] = [22.22222222222222, 26.25, 27.142857142857146, 30.0] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_mul_Marks"] = [18.0, 16.8, 13.3, 10.8] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Age_div_Marks": [22.22222222222222, 26.25, 27.142857142857146, 30.0], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Age_mul_Marks": [18.0, 16.8, 13.299999999999999, 10.799999999999999], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -def test_alternative_operation(df_vartypes): + assert_df_equal(Xt, expected) - # input df - df = df_vartypes.copy() - - # Expected result - dft = df.copy() - dft["Age_truediv_Marks"] = dft["Age"].truediv(dft["Marks"]) - dft["Age_floordiv_Marks"] = dft["Age"].floordiv(dft["Marks"]) - dft["Age_mod_Marks"] = dft["Age"].mod(dft["Marks"]) - dft["Age_pow_Marks"] = dft["Age"].pow(dft["Marks"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_alternative_operation(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=["truediv", "floordiv", "mod", "pow"], ) - X = transformer.fit_transform(df) + Xt = transformer.fit_transform(df) + + expected = dict(DATA) + expected["Age_truediv_Marks"] = [22.22222222222222, 26.25, 27.142857142857146, 30.0] + expected["Age_floordiv_Marks"] = [22.0, 26.0, 27.0, 30.0] + expected["Age_mod_Marks"] = [ + 0.1999999999999995, + 0.19999999999999885, + 0.1000000000000012, + 6.661338147750939e-16, + ] + expected["Age_pow_Marks"] = [ + 14.822688982138954, + 11.42287530066645, + 7.85466234994081, + 5.664525067769412, + ] - pd.testing.assert_frame_equal(X, dft) + assert_df_equal(Xt, expected) -def test_operations_with_multiple_variables(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_operations_with_multiple_variables(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["sub"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) + assert_df_equal(Xt, expected) - pd.testing.assert_frame_equal(X, ref) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_multiple_operations_with_multiple_variables(make_df): + df = make_df(DATA) -def test_multiple_operations_with_multiple_variables(df_vartypes): + # column order follows func order: sub's 4 columns, then add's 4 transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["sub", "add"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - } - ) + expected = dict(DATA) + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + expected["Age_add_Age"] = [40, 42, 38, 36] + expected["Marks_add_Age"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Marks_add_Marks"] = [1.8, 1.6, 1.4, 1.2] - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) + # reversing func order reverses the corresponding column block order transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) + expected = dict(DATA) + expected["Age_add_Age"] = [40, 42, 38, 36] + expected["Marks_add_Age"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Marks_add_Marks"] = [1.8, 1.6, 1.4, 1.2] + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + assert_df_equal(Xt, expected) -def test_when_missing_values_is_ignore(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_when_missing_values_is_ignore(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) transformer = RelativeFeatures( variables=["Age", "Marks"], @@ -255,30 +263,22 @@ def test_when_missing_values_is_ignore(df_vartypes): func=["sub"], missing_values="ignore", ) + Xt = transformer.fit_transform(df_na) - X = transformer.fit_transform(df_na) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, np.nan, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, np.nan, 0, 0], - "Marks_sub_Age": [-19.1, np.nan, -18.3, -17.4], - "Age_sub_Marks": [19.1, np.nan, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) + expected = dict(data_na) + expected["Age_sub_Age"] = [0, np.nan, 0, 0] + expected["Marks_sub_Age"] = [-19.1, np.nan, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, np.nan, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + assert_df_equal(Xt, expected) -def test_error_when_null_values_in_variable(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_null_values_in_variable(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) transformer = RelativeFeatures( variables=["Age", "Marks"], @@ -290,14 +290,16 @@ def test_error_when_null_values_in_variable(df_vartypes): with pytest.raises(ValueError): transformer.fit(df_na) - transformer.fit(df_vartypes) + transformer.fit(make_df(DATA)) with pytest.raises(ValueError): transformer.transform(df_na) -def test_when_df_cols_are_integers(df_vartypes): - df = df_vartypes.copy() - df.columns = [0, 1, 2, 3, 4] +def test_when_df_cols_are_integers(): + # polars requires string column names, so int-named columns are + # pandas-only - no polars equivalent to parametrize against here. + df = pd.DataFrame(DATA) + df.columns = [0, 1, 2, 3] transformer = RelativeFeatures( variables=[2, 3], @@ -313,7 +315,6 @@ def test_when_df_cols_are_integers(df_vartypes): 1: ["London", "Manchester", "Liverpool", "Bristol"], 2: [20, 21, 19, 18], 3: [0.9, 0.8, 0.7, 0.6], - 4: pd.date_range("2020-02-24", periods=4, freq="min"), "2_sub_2": [0, 0, 0, 0], "3_sub_2": [-19.1, -20.2, -18.3, -17.4], "2_sub_3": [19.1, 20.2, 18.3, 17.4], @@ -328,31 +329,30 @@ def test_when_df_cols_are_integers(df_vartypes): pd.testing.assert_frame_equal(X, ref) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_func", [["div"], ["truediv"], ["floordiv"], ["mod"]]) -def test_error_when_division_by_zero_and_fill_value_is_none(_func, df_vartypes): - - df_zero = df_vartypes.copy() - df_zero.loc[1, "Marks"] = 0 +def test_error_when_division_by_zero_and_fill_value_is_none(make_df, _func): + data_zero = dict(DATA) + data_zero["Marks"] = [0.9, 0, 0.7, 0.6] + df_zero = make_df(data_zero) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=_func, ) - transformer.fit(df_vartypes) - - with pytest.raises(ValueError) as record: - transformer.transform(df_zero) + transformer.fit(make_df(DATA)) msg = ( "Some of the reference variables contain zeroes. Division by zero " "does not exist. Replace zeros before using this transformer for division " "or set `fill_value` to a number." ) - # check that the error message matches - assert str(record.value) == msg + with pytest.raises(ValueError, match=msg): + transformer.transform(df_zero) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "_fill_value, _func", [ @@ -366,11 +366,13 @@ def test_error_when_division_by_zero_and_fill_value_is_none(_func, df_vartypes): (999, ["mod"]), ], ) -def test_fill_values_when_division_by_zero(_fill_value, _func, df_vartypes): - df_zero = df_vartypes.copy() - df_zero.loc[2, "Marks"] = 0 - df_zero.loc[1, "Age"] = np.nan - df_zero.loc[3, "Age"] = np.inf +def test_fill_values_when_division_by_zero(make_df, _fill_value, _func): + data_zero = dict(DATA) + data_zero["Marks"] = [0.9, 0.8, 0, 0.6] + # Age must be float from the start: polars can't build an Int64 column + # from a mix of ints and NaN/inf the way pandas silently upcasts to. + data_zero["Age"] = [20.0, np.nan, 19.0, np.inf] + df_zero = make_df(data_zero) transformer = RelativeFeatures( variables=["Age"], @@ -379,18 +381,20 @@ def test_fill_values_when_division_by_zero(_fill_value, _func, df_vartypes): func=_func, missing_values="ignore", ) - - X = transformer.fit_transform(df_zero) + Xt = transformer.fit_transform(df_zero) new_var = f"Age_{_func[0]}_Marks" + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) - assert X.loc[2, new_var] == _fill_value - np.testing.assert_equal(X.loc[1, "Age"], np.nan) - np.testing.assert_equal(X.loc[3, "Age"], np.inf) + assert result[new_var][2] == pytest.approx(_fill_value) + np.testing.assert_equal(result["Age"][1], np.nan) + np.testing.assert_equal(result["Age"][3], np.inf) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out(_drop, df_vartypes): +def test_get_feature_names_out(make_df, _drop): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], @@ -407,55 +411,88 @@ def test_get_feature_names_out(_drop, df_vartypes): "Age_sub_Marks", "Marks_sub_Marks", ] - X = transformer.fit_transform(df_vartypes) - feat_out = list(X.columns) + Xt = transformer.fit_transform(df) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) assert feat_out == transformer.get_feature_names_out(input_features=None) - assert feat_out == transformer.get_feature_names_out( - input_features=df_vartypes.columns - ) assert all([f for f in varnames if f in feat_out]) + if _drop is True: + # drop_original only drops columns that are in variables/reference + # (here Age, Marks) - Name and City are neither, so they remain. + assert feat_out == ["Name", "City"] + varnames + else: + assert feat_out == list(DATA.keys()) + varnames +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out_from_pipeline(_drop, df_vartypes): +def test_get_feature_names_out_from_pipeline(make_df, _drop): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], drop_original=_drop, ) - pipe = Pipeline([("transformer", transformer)]) - varnames = [ - "Age_add_Age", - "Marks_add_Age", - "Age_add_Marks", - "Marks_add_Marks", - "Age_sub_Age", - "Marks_sub_Age", - "Age_sub_Marks", - "Marks_sub_Marks", - ] - - X = pipe.fit_transform(df_vartypes) - assert list(X.columns) == pipe.get_feature_names_out(input_features=None) - assert list(X.columns) == pipe.get_feature_names_out( - input_features=df_vartypes.columns - ) - assert all([f for f in varnames if f in X.columns]) + Xt = pipe.fit_transform(df) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) + assert feat_out == pipe.get_feature_names_out(input_features=None) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) -def test_get_feature_names_out_raises_error_when_wrong_param( - _input_features, df_vartypes -): +def test_get_feature_names_out_raises_error_when_wrong_param(make_df, _input_features): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], ) - transformer.fit(df_vartypes) + transformer.fit(df) with pytest.raises(ValueError): transformer.get_feature_names_out(input_features=_input_features) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_mixed_int_float_variables_preserve_own_dtype(make_df): + # a regression check: extracting variables as one batched 2D array + # upcasts everything to a common dtype, losing e.g. an int column's + # own int result for subtraction. Each variable must keep its own + # dtype promotion, independent of the other variables in the list. + df = make_df(DATA) + transformer = RelativeFeatures( + variables=["Age", "Marks"], reference=["Age"], func=["sub"] + ) + Xt = transformer.fit_transform(df) + nw_Xt = nw.from_native(Xt, eager_only=True) + assert nw_Xt.get_column("Age_sub_Age").dtype.is_integer() + assert not nw_Xt.get_column("Marks_sub_Age").dtype.is_integer() + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_floordiv_zero_with_float_fill_value_widens_dtype(make_df): + # floordiv on integer input stays integer-typed; a float fill_value + # must widen the result column rather than truncating or erroring, + # matching pandas' own automatic dtype promotion here. + df = make_df({"v": [7, 8], "ref": [0, 2]}) + transformer = RelativeFeatures( + variables=["v"], reference=["ref"], func=["floordiv"], fill_value=-1.5 + ) + Xt = transformer.fit_transform(df) + result = nw.from_native(Xt, eager_only=True).get_column("v_floordiv_ref").to_list() + assert result == pytest.approx([-1.5, 4.0]) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_drop_original_both_backends(make_df): + df = make_df({"x1": [1, 2, 3], "x2": [4, 5, 6], "x3": [3, 4, 5]}) + transformer = RelativeFeatures( + variables=["x1", "x2"], reference=["x3"], func=["div"], drop_original=True + ) + Xt = transformer.fit_transform(df) + assert list(nw.from_native(Xt, eager_only=True).columns) == [ + "x1_div_x3", + "x2_div_x3", + ]