diff --git a/.circleci/config.yml b/.circleci/config.yml index 127b33bef..0d31c7f44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ orbs: defaults: &defaults docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project prepare_tox: &prepare_tox @@ -42,58 +42,6 @@ jobs: # Test matrix # ------------------------ - test_feature_engine_py39: - docker: - - image: cimg/python:3.9.0 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.9) - command: | - tox -e py39 - - test_feature_engine_py310: - docker: - - image: cimg/python:3.10.0 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.10) - command: | - tox -e py310 - - test_feature_engine_py311_sklearn150: - docker: - - image: cimg/python:3.11.7 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.11, scikit-learn 1.5) - command: | - tox -e py311-sklearn150 - - test_feature_engine_py311_sklearn160: - docker: - - image: cimg/python:3.11.7 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.11, scikit-learn 1.6) - command: | - tox -e py311-sklearn160 - test_feature_engine_py311_sklearn170: docker: - image: cimg/python:3.11.7 @@ -166,7 +114,7 @@ jobs: test_style: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -179,7 +127,7 @@ jobs: test_docs: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -192,7 +140,7 @@ jobs: test_type: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -277,10 +225,6 @@ workflows: test-all: jobs: - - test_feature_engine_py39 - - test_feature_engine_py310 - - test_feature_engine_py311_sklearn150 - - test_feature_engine_py311_sklearn160 - test_feature_engine_py311_sklearn170 - test_feature_engine_py312_pandas230 - test_feature_engine_py312_pandas300 @@ -298,10 +242,6 @@ workflows: - package_and_upload_to_pypi: requires: - - test_feature_engine_py39 - - test_feature_engine_py310 - - test_feature_engine_py311_sklearn150 - - test_feature_engine_py311_sklearn160 - test_feature_engine_py311_sklearn170 - test_feature_engine_py312_pandas230 - test_feature_engine_py312_pandas300 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..dd3524880 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,101 @@ +# AGENTS.md + +Conventions for working in this repo. Optimize for readability and speed, +in that order of how you decide, but don't ship a slow default when a +fast one is free. + +## Inputs + +Feature-engine transformers take dataframes (pandas, polars, or any other +narwhals-supported backend) as input, not numpy arrays. Don't add +handling for array input. + +## Never import pandas in library code + +pandas is an optional dependency (see `pyproject.toml` — it lives under +`[project.optional-dependencies]`, not core `dependencies`), so `import +pandas` must never appear anywhere in `feature_engine/`, not at module level +and not locally/lazily inside a function either — importing the module +itself would break a polars-only install regardless of which class is used. + +Backend checks go through `narwhals.dependencies` (`nwd.is_pandas_dataframe`, +`nwd.is_pandas_series`, `nwd.is_pandas_index`, `nwd.is_into_series`, etc.). +Once a branch is confirmed pandas, call its methods/attributes directly on +the object already in hand (`.loc`, `.columns`, `.index`, `.select_dtypes`, +...) — no import needed for that, since Python only needs a module imported +to reference the module itself (`pd.something`), not to call methods on an +object that's already an instance of that module's class. + +## Booleans and control flow + +- Compare booleans explicitly: `if x is True:` / `if x is False:`, never + `if x:` / `if not x:`. +- Check container emptiness with `len(x) == 0`, never `if not x:`. +- `isinstance(...)` checks and `in`/`not in` membership tests are already + explicit — leave them as-is, this rule isn't about those. +- The explicit `is True`/`is False` comparison is for flow control + (`if`/`while` conditions) only — don't tack it onto a variable + assignment. When a function already returns a strict bool (e.g. + `nwd.is_pandas_dataframe(X)`), assign it directly: + `is_pandas = nwd.is_pandas_dataframe(X)`, not + `is_pandas = nwd.is_pandas_dataframe(X) is True`. The `if`/`while` site + that later consumes `is_pandas` still spells out `if is_pandas is True:`. + +## Comments + +Max 2 lines. Only explain a non-obvious WHY (a hidden constraint, a subtle +backend difference, a workaround) — never describe WHAT the code does. + +## Don't anticipate errors + +Don't add error handling or validation for scenarios that can't happen. If +unsure whether something can happen, check it (grep, run a quick repro) or +ask — don't guess and defensively code around it. + +## Redundant lists/sets + +- Narwhals' `.columns` is already `list[str]` — don't wrap it in `list()`. +- pandas' `.columns` is an `Index`, not a list — `list()` is required there + (an `Index == list` comparison is elementwise, not a clean bool). + +## Keep tests passing when you change a function or class + +Whenever you change a function or class, run its corresponding tests. If +they fail, resolve it — don't leave it — by figuring out whether the test +needs updating (e.g. it exercised behavior that's no longer supported) or +the implementation has a real bug, and fixing whichever one is wrong. + +## Keep docs in sync with transformer changes + +When new functionality is introduced in a transformer, update its +corresponding `docs/user_guide//.rst` with a short +worked example showing the new functionality. + +## Verify before applying + +Benchmark before claiming a speedup, and diff old-vs-new output across +realistic and edge cases (empty/all-NaN, both backends, both dtype +branches) before trusting a rewrite — logic mistakes here are easy to make +and easy to miss without an actual comparison. + +## Tests + +- `pytest.raises(ExceptionType, match=msg)`, never + `with pytest.raises() as record: ... assert str(record.value) == msg`. +- Dataframe-agnostic means one test, both backends: parametrize each + behavior over `@pytest.mark.parametrize("make_df", [pd.DataFrame, + pl.DataFrame])` and assert the same input produces the same output + values on both. Never write a separate pandas-only test and a + separate polars-only test for the same behavior — that duplicates + the test and hides the point of being dataframe-agnostic, which is + that the same input gives the same output regardless of backend. + Keep a test single-backend only when the behavior itself is + backend-specific (e.g. integer column names, which polars doesn't + support; pandas nullable extension dtypes). + +## API changes + +- New parameters default to preserve current behavior. +- When adding a parameter to a function called from multiple sites (or a + shared private helper), thread it through every call site, not just the + one you're looking at. diff --git a/README.md b/README.md index 8f04b87e0..3622ba9a3 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ Feature-engine documentation is built using [Sphinx](https://www.sphinx-doc.org) To build the documentation make sure you have the dependencies installed: from the root directory: ``` -pip install -r docs/requirements.txt +pip install -e ".[docs]" ``` Now you can build the docs using: diff --git a/docs/contribute/contribute_code.rst b/docs/contribute/contribute_code.rst index 4e85515ec..d5413e051 100644 --- a/docs/contribute/contribute_code.rst +++ b/docs/contribute/contribute_code.rst @@ -395,7 +395,7 @@ To do this, first make sure you have all the documentation dependencies installe set up the environment as we described previously, they should be installed. Alternatively, from the windows cmd or mac terminal, run:: - $ pip install -r docs/requirements.txt + $ pip install -e ".[docs]" Make sure you are within the feature_engine module when you run the previous command. diff --git a/docs/contribute/contribute_docs.rst b/docs/contribute/contribute_docs.rst index 856e22d85..ee16ed9f4 100644 --- a/docs/contribute/contribute_docs.rst +++ b/docs/contribute/contribute_docs.rst @@ -79,7 +79,7 @@ dependencies. If you set up the development environment as we described in the Alternatively, first activate your environment. Then navigate to the root folder of feature-engine. And now install the requirements for the documentation:: - $ pip install -r docs/requirements.txt + $ pip install -e ".[docs]" To build the documentation (and test if it is working properly) run:: diff --git a/docs/index.rst b/docs/index.rst index 98e8a3bac..2d7931cc9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -127,7 +127,7 @@ The following characteristics make feature-engine unique: Installation ------------ -Feature-engine is a Python 3 package and works well with 3.9 or later. +Feature-engine is a Python 3 package and works well with 3.11 or later. The simplest way to install feature-engine is from PyPI with pip: diff --git a/docs/user_guide/creation/CyclicalFeatures.rst b/docs/user_guide/creation/CyclicalFeatures.rst index 59b26567a..5226afe23 100644 --- a/docs/user_guide/creation/CyclicalFeatures.rst +++ b/docs/user_guide/creation/CyclicalFeatures.rst @@ -208,6 +208,60 @@ This returns the name of all the variables in the final output: ['day_sin', 'day_cos', 'months_sin', 'months_cos'] +With polars +----------- + +:class:`CyclicalFeatures()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataframe: + +.. code:: python + + import polars as pl + from feature_engine.creation import CyclicalFeatures + + df = pl.DataFrame({ + "day": [6, 7, 5, 3, 1, 2, 4], + "months": [3, 7, 9, 12, 4, 6, 12], + }) + + cyclical = CyclicalFeatures(variables=None, drop_original=False) + X = cyclical.fit_transform(df) + + cyclical.max_values_ + +The maximum values match those found with pandas: + +.. code:: python + + {'day': 7, 'months': 12} + +And the transformed dataframe contains the same cyclical features: + +.. code:: python + + print(X) + +.. code:: text + + shape: (7, 6) + ┌─────┬────────┬─────────────┬───────────┬─────────────┬─────────────┐ + │ day ┆ months ┆ day_sin ┆ day_cos ┆ months_sin ┆ months_cos │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ i64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ + ╞═════╪════════╪═════════════╪═══════════╪═════════════╪═════════════╡ + │ 6 ┆ 3 ┆ -0.781831 ┆ 0.62349 ┆ 1.0 ┆ 6.1232e-17 │ + │ 7 ┆ 7 ┆ -2.4493e-16 ┆ 1.0 ┆ -0.5 ┆ -0.866025 │ + │ 5 ┆ 9 ┆ -0.974928 ┆ -0.222521 ┆ -1.0 ┆ -1.8370e-16 │ + │ 3 ┆ 12 ┆ 0.433884 ┆ -0.900969 ┆ -2.4493e-16 ┆ 1.0 │ + │ 1 ┆ 4 ┆ 0.781831 ┆ 0.62349 ┆ 0.866025 ┆ -0.5 │ + │ 2 ┆ 6 ┆ 0.974928 ┆ -0.222521 ┆ 1.2246e-16 ┆ -1.0 │ + │ 4 ┆ 12 ┆ -0.433884 ┆ -0.900969 ┆ -2.4493e-16 ┆ 1.0 │ + └─────┴────────┴─────────────┴───────────┴─────────────┴─────────────┘ + +`drop_original=True` and `get_feature_names_out()` work identically to the +pandas example above. + + Understanding cyclical encoding ------------------------------- diff --git a/docs/user_guide/creation/DecisionTreeFeatures.rst b/docs/user_guide/creation/DecisionTreeFeatures.rst index 56c6a4056..ba59800d2 100644 --- a/docs/user_guide/creation/DecisionTreeFeatures.rst +++ b/docs/user_guide/creation/DecisionTreeFeatures.rst @@ -485,6 +485,53 @@ are not there: 2670 1.843904 15709 1.843904 +With polars +----------- + +:class:`DecisionTreeFeatures()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.creation import DecisionTreeFeatures + + X = pl.DataFrame({ + "Age": [20, 44, 19, 33, 51, 40, 41, 37, 30, 54], + "Height": [164, 150, 178, 158, 188, 190, 168, 174, 176, 171], + }) + y = [4.1, 5.8, 3.9, 6.2, 4.3, 4.5, 7.2, 4.4, 4.1, 6.7] + + dtf = DecisionTreeFeatures(features_to_combine=2, drop_original=True) + dtf.fit(X, y) + + print(dtf.transform(X)) + +The resulting values match those found with pandas: + +.. code:: text + + shape: (10, 3) + ┌───────────┬──────────────┬─────────────────────────┐ + │ tree(Age) ┆ tree(Height) ┆ tree(['Age', 'Height']) │ + │ --- ┆ --- ┆ --- │ + │ f64 ┆ f64 ┆ f64 │ + ╞═══════════╪══════════════╪═════════════════════════╡ + │ 4.533333 ┆ 5.366667 ┆ 4.1 │ + │ 6.0 ┆ 5.366667 ┆ 6.475 │ + │ 4.533333 ┆ 4.133333 ┆ 4.0 │ + │ 4.533333 ┆ 5.366667 ┆ 6.475 │ + │ 6.0 ┆ 4.4 ┆ 4.4 │ + │ 4.533333 ┆ 4.4 ┆ 4.4 │ + │ 6.0 ┆ 6.95 ┆ 6.475 │ + │ 4.533333 ┆ 4.133333 ┆ 4.4 │ + │ 4.533333 ┆ 4.133333 ┆ 4.0 │ + │ 6.0 ┆ 6.95 ┆ 6.475 │ + └───────────┴──────────────┴─────────────────────────┘ + +`get_feature_names_out()`, classification, and every other parameter shown +above with pandas work identically with polars. + + Creating features for classification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -498,6 +545,46 @@ identical. We just need to set the parameter `regression` to False. classification, on the other hand, the features will contain the prediction of the class. +Training trees in parallel +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each tree is trained on its own feature combination independently of the others, so +when there are many combinations (a large number of variables and/or a high +`features_to_combine`) or a large `param_grid` to search, training can be +parallelized across combinations with the `n_jobs` parameter: + +.. code:: python + + import pandas as pd + from feature_engine.creation import DecisionTreeFeatures + + X = pd.DataFrame({ + "Age": [20, 44, 19, 33, 51, 40, 41, 37, 30, 54], + "Height": [164, 150, 178, 158, 188, 190, 168, 174, 176, 171], + "Marks": [1.0, 0.8, 0.6, 0.1, 0.3, 0.4, 0.8, 0.6, 0.5, 0.2], + }) + y = [4.1, 5.8, 3.9, 6.2, 4.3, 4.5, 7.2, 4.4, 4.1, 6.7] + + dtf = DecisionTreeFeatures(features_to_combine=3, n_jobs=2, random_state=0) + dtf.fit(X, y) + + print(dtf.transform(X).columns.tolist()) + +.. code:: text + + ['Age', 'Height', 'Marks', 'tree(Age)', 'tree(Height)', 'tree(Marks)', + "tree(['Age', 'Height'])", "tree(['Age', 'Marks'])", + "tree(['Height', 'Marks'])", "tree(['Age', 'Height', 'Marks'])"] + +`n_jobs` defaults to `None`, which trains the trees sequentially, matching this +transformer's original behaviour. Setting it trains multiple trees at the same +time using threads, which only pays off once there are enough feature +combinations or a large enough `param_grid` to outweigh the overhead of +dispatching work to threads — with just a handful of combinations, sequential +training is faster. The resulting trees and predictions are identical +regardless of `n_jobs`; only training speed changes. + + Additional resources -------------------- diff --git a/docs/user_guide/creation/GeoDistanceFeatures.rst b/docs/user_guide/creation/GeoDistanceFeatures.rst index 9744d61e9..c142c4009 100644 --- a/docs/user_guide/creation/GeoDistanceFeatures.rst +++ b/docs/user_guide/creation/GeoDistanceFeatures.rst @@ -77,11 +77,11 @@ In the following output we see the trip ID followed by the distance travelled in .. code:: python - trip_id distance_km - 0 1 3935.746254 - 1 2 2808.517344 - 2 3 1144.286561 - 3 4 1634.724892 + trip_id distance_km + 0 1 3935.746255 + 1 2 2803.971507 + 2 3 1144.291274 + 3 4 1632.166882 Using different distance methods ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -108,10 +108,10 @@ for Earth's curvature: .. code:: python trip_id distance_euclidean - 0 1 4940.252715 - 1 2 3493.298968 - 2 3 1519.295694 - 3 4 1720.178310 + 0 1 4965.730734 + 1 2 3507.416606 + 2 3 1517.763567 + 3 4 1898.819227 Alternatively, we can use the Manhattan distance, which is useful for grid-based city layouts: @@ -133,10 +133,10 @@ The Manhattan distance sums the absolute differences in latitude and longitude: .. code:: python trip_id distance_manhattan - 0 1 5628.24000 - 1 2 4684.15800 - 2 3 1637.36700 - 3 4 2279.96460 + 0 1 5649.7113 + 1 2 4266.8178 + 2 3 1641.5901 + 3 4 2263.5342 Using different output units ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -162,10 +162,10 @@ The distances are now expressed in miles instead of kilometres: .. code:: python trip_id distance_miles - 0 1 2445.258392 - 1 2 1745.046817 - 2 3 711.000629 - 3 4 1015.643614 + 0 1 2445.586607 + 1 2 1742.326542 + 2 3 711.037560 + 3 4 1014.192788 Dropping original coordinate columns ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -193,6 +193,55 @@ After transformation, only the non-coordinate columns and the new distance colum ['trip_id', 'geo_distance'] +With polars +----------- + +:class:`GeoDistanceFeatures()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from feature_engine.creation import GeoDistanceFeatures + + X = pl.DataFrame({ + 'origin_lat': [40.7128, 34.0522, 41.8781, 29.7604], + 'origin_lon': [-74.0060, -118.2437, -87.6298, -95.3698], + 'dest_lat': [34.0522, 41.8781, 40.7128, 33.4484], + 'dest_lon': [-118.2437, -87.6298, -74.0060, -112.0740], + 'trip_id': [1, 2, 3, 4] + }) + + gdt = GeoDistanceFeatures( + lat1='origin_lat', lon1='origin_lon', + lat2='dest_lat', lon2='dest_lon', + method='haversine', output_unit='km', output_col='distance_km' + ) + + gdt.fit(X) + X_transformed = gdt.transform(X) + + print(X_transformed.select(['trip_id', 'distance_km'])) + +We see the resulting distances: + +.. code:: text + + shape: (4, 2) + ┌─────────┬─────────────┐ + │ trip_id ┆ distance_km │ + │ --- ┆ --- │ + │ i64 ┆ f64 │ + ╞═════════╪═════════════╡ + │ 1 ┆ 3935.746255 │ + │ 2 ┆ 2803.971507 │ + │ 3 ┆ 1144.291274 │ + │ 4 ┆ 1632.166882 │ + └─────────┴─────────────┘ + +`drop_original=True` and the different distance methods and output units +work identically to the pandas examples above. + Calculating distance within a Pipeline ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -232,7 +281,7 @@ The pipeline successfully trains and returns predictions: .. code:: python - Predictions: [100. 150. 80. 200.] + Predictions: [116.67298659 120.75252844 88.47598336 204.09850161] Additional resources -------------------- diff --git a/docs/user_guide/creation/MathFeatures.rst b/docs/user_guide/creation/MathFeatures.rst index 6b79af525..0c119d348 100644 --- a/docs/user_guide/creation/MathFeatures.rst +++ b/docs/user_guide/creation/MathFeatures.rst @@ -143,11 +143,11 @@ We obtain the following dataframe: 2 krish Liverpool 19 0.7 2020-02-24 00:02:00 19.7 3 jack Bristol 18 0.6 2020-02-24 00:03:00 18.6 - prod_Age_Marks amin_Age_Marks amax_Age_Marks std_Age_Marks - 0 18.0 0.9 20.0 13.505740 - 1 16.8 0.8 21.0 14.283557 - 2 13.3 0.7 19.0 12.940054 - 3 10.8 0.6 18.0 12.303658 + prod_Age_Marks min_Age_Marks max_Age_Marks std_Age_Marks + 0 18.0 0.9 20.0 9.55 + 1 16.8 0.8 21.0 10.10 + 2 13.3 0.7 19.0 9.15 + 3 10.8 0.6 18.0 8.70 We have the option to set the parameter `drop_original` to True to drop the variables after performing the calculations. @@ -169,11 +169,60 @@ Which will return the names of all the variables in the transformed data: 'dob', 'sum_Age_Marks', 'prod_Age_Marks', - 'amin_Age_Marks', - 'amax_Age_Marks', + 'min_Age_Marks', + 'max_Age_Marks', 'std_Age_Marks'] +With polars +----------- + +:class:`MathFeatures()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.creation import MathFeatures + + df = pl.DataFrame({ + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + }) + + transformer = MathFeatures( + variables=["Age", "Marks"], + func=["sum", "prod", "min", "max", "std"], + ) + + print(transformer.fit_transform(df)) + +The resulting values match those found with pandas: + +.. code:: text + + shape: (4, 7) + ┌─────┬───────┬───────────────┬────────────────┬───────────────┬───────────────┬───────────────┐ + │ Age ┆ Marks ┆ sum_Age_Marks ┆ prod_Age_Marks ┆ min_Age_Marks ┆ max_Age_Marks ┆ std_Age_Marks │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ + ╞═════╪═══════╪═══════════════╪════════════════╪═══════════════╪═══════════════╪═══════════════╡ + │ 20 ┆ 0.9 ┆ 20.9 ┆ 18.0 ┆ 0.9 ┆ 20.0 ┆ 13.50574 │ + │ 21 ┆ 0.8 ┆ 21.8 ┆ 16.8 ┆ 0.8 ┆ 21.0 ┆ 14.283557 │ + │ 19 ┆ 0.7 ┆ 19.7 ┆ 13.3 ┆ 0.7 ┆ 19.0 ┆ 12.940054 │ + │ 18 ┆ 0.6 ┆ 18.6 ┆ 10.8 ┆ 0.6 ┆ 18.0 ┆ 12.303658 │ + └─────┴───────┴───────────────┴────────────────┴───────────────┴───────────────┴───────────────┘ + +`new_variables_names`, `drop_original`, and `get_feature_names_out()` work +identically to the pandas examples above. + +If you pass a custom Python callable as `func` (instead of a string or one +of the common aggregations above, which are always NumPy-vectorized), note +that the callable receives a **plain tuple** of values for polars input, +not a pandas `Series` — so `lambda row: max(row) - min(row)` works on both +backends, but `lambda row: row.max() - row.min()` (which relies on `Series` +methods) only works with pandas. + + New variables names ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/user_guide/creation/RelativeFeatures.rst b/docs/user_guide/creation/RelativeFeatures.rst index 5611aa204..870b5c1cd 100644 --- a/docs/user_guide/creation/RelativeFeatures.rst +++ b/docs/user_guide/creation/RelativeFeatures.rst @@ -141,6 +141,51 @@ Which will return the names of all the variables in the transformed data: 'Marks_pow_Age'] +With polars +----------- + +:class:`RelativeFeatures()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.creation import RelativeFeatures + + df = pl.DataFrame({ + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + }) + + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age"], + func = ["sub", "div", "mod", "pow"], + ) + + print(transformer.fit_transform(df)) + +The resulting values match those found with pandas (`Age_pow_Age`'s large +values are genuine `int64` overflow from raising `Age` to the power of +itself, not an error - the same happens with pandas): + +.. code:: text + + shape: (4, 10) + ┌─────┬───────┬─────────────┬───────────────┬─────────────┬───────────────┬─────────────┬───────────────┬──────────────────────┬───────────────┐ + │ Age ┆ Marks ┆ Age_sub_Age ┆ Marks_sub_Age ┆ Age_div_Age ┆ Marks_div_Age ┆ Age_mod_Age ┆ Marks_mod_Age ┆ Age_pow_Age ┆ Marks_pow_Age │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ i64 ┆ f64 ┆ f64 ┆ f64 ┆ i64 ┆ f64 ┆ i64 ┆ f64 │ + ╞═════╪═══════╪═════════════╪═══════════════╪═════════════╪═══════════════╪═════════════╪═══════════════╪══════════════════════╪═══════════════╡ + │ 20 ┆ 0.9 ┆ 0 ┆ -19.1 ┆ 1.0 ┆ 0.045 ┆ 0 ┆ 0.9 ┆ -2101438300051996672 ┆ 0.121577 │ + │ 21 ┆ 0.8 ┆ 0 ┆ -20.2 ┆ 1.0 ┆ 0.038095 ┆ 0 ┆ 0.8 ┆ -1595931050845505211 ┆ 0.009223 │ + │ 19 ┆ 0.7 ┆ 0 ┆ -18.3 ┆ 1.0 ┆ 0.036842 ┆ 0 ┆ 0.7 ┆ 6353754964178307979 ┆ 0.00114 │ + │ 18 ┆ 0.6 ┆ 0 ┆ -17.4 ┆ 1.0 ┆ 0.033333 ┆ 0 ┆ 0.6 ┆ -497033925936021504 ┆ 0.000102 │ + └─────┴───────┴─────────────┴───────────────┴─────────────┴───────────────┴─────────────┴───────────────┴──────────────────────┴───────────────┘ + +`fill_value`, `drop_original`, and `get_feature_names_out()` work +identically to the pandas examples above. + + Additional resources -------------------- diff --git a/docs/user_guide/transformation/ArcSinhTransformer.rst b/docs/user_guide/transformation/ArcSinhTransformer.rst index 36dd44862..39806e94a 100644 --- a/docs/user_guide/transformation/ArcSinhTransformer.rst +++ b/docs/user_guide/transformation/ArcSinhTransformer.rst @@ -553,6 +553,44 @@ The recovered data: 493 -3.258723 9405.785347 122 30.047946 1448.874284 +With polars +----------- + +:class:`ArcSinhTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import ArcSinhTransformer + + df = pl.DataFrame({ + "profit": [12.14, 6.43, 14.12, 33.89, 5.85, -2.5, 0.0], + "net_worth": [-8516.91, -277.74, 1920.33, -163.47, -10337.21, 500.0, 0.0], + }) + + tf = ArcSinhTransformer(variables=["profit", "net_worth"]) + tf.fit(df) + Xt = tf.transform(df) + + print(Xt) + +.. code:: text + + shape: (7, 2) + ┌───────────┬───────────┐ + │ profit ┆ net_worth │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞═══════════╪═══════════╡ + │ 3.191345 ┆ -9.742956 │ + │ 2.560114 ┆ -6.319836 │ + │ 3.341991 ┆ 8.2534 │ + │ 4.216485 ┆ -5.789786 │ + │ 2.466815 ┆ -9.936652 │ + │ -1.647231 ┆ 6.907756 │ + │ 0.0 ┆ 0.0 │ + └───────────┴───────────┘ + References ---------- diff --git a/docs/user_guide/transformation/ArcsinTransformer.rst b/docs/user_guide/transformation/ArcsinTransformer.rst index d69fa20ff..5d82e1280 100644 --- a/docs/user_guide/transformation/ArcsinTransformer.rst +++ b/docs/user_guide/transformation/ArcsinTransformer.rst @@ -134,6 +134,42 @@ shape after the transformation: .. image:: ../../images/breast_cancer_arcsin.png +With polars +----------- + +:class:`ArcsinTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import ArcsinTransformer + + df = pl.DataFrame({ + "proportion_1": [0.1, 0.2, 0.3, 0.4, 0.5], + "proportion_2": [0.9, 0.8, 0.7, 0.6, 0.5], + }) + + tf = ArcsinTransformer(variables=None) + tf.fit(df) + Xt = tf.transform(df) + + print(Xt) + +.. code:: text + + shape: (5, 2) + ┌──────────────┬──────────────┐ + │ proportion_1 ┆ proportion_2 │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞══════════════╪══════════════╡ + │ 0.321751 ┆ 1.249046 │ + │ 0.463648 ┆ 1.107149 │ + │ 0.57964 ┆ 0.991157 │ + │ 0.684719 ┆ 0.886077 │ + │ 0.785398 ┆ 0.785398 │ + └──────────────┴──────────────┘ + Additional resources -------------------- diff --git a/docs/user_guide/transformation/BoxCoxTransformer.rst b/docs/user_guide/transformation/BoxCoxTransformer.rst index 5340d88b1..ad3b66043 100644 --- a/docs/user_guide/transformation/BoxCoxTransformer.rst +++ b/docs/user_guide/transformation/BoxCoxTransformer.rst @@ -206,6 +206,43 @@ In the following plots we see that the variables are non-normally distributed, b .. image:: ../../images/nonnormalvars2.png +With polars +----------- + +:class:`BoxCoxTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import BoxCoxTransformer + + df = pl.DataFrame({ + "var_1": [4.0, 9.0, 16.0, 25.0, 100.0], + "var_2": [1.0, 8.0, 27.0, 64.0, 125.0], + }) + + boxcox = BoxCoxTransformer(variables=None) + boxcox.fit(df) + Xt = boxcox.transform(df) + + print(Xt) + +.. code:: text + + shape: (5, 2) + ┌──────────┬──────────┐ + │ var_1 ┆ var_2 │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞══════════╪══════════╡ + │ 1.225161 ┆ 0.0 │ + │ 1.810873 ┆ 2.666746 │ + │ 2.177001 ┆ 4.93175 │ + │ 2.435721 ┆ 6.969848 │ + │ 3.117497 ┆ 8.854291 │ + └──────────┴──────────┘ + + Additional resources -------------------- diff --git a/docs/user_guide/transformation/LogCpTransformer.rst b/docs/user_guide/transformation/LogCpTransformer.rst index 0f90f85af..b4b224d86 100644 --- a/docs/user_guide/transformation/LogCpTransformer.rst +++ b/docs/user_guide/transformation/LogCpTransformer.rst @@ -93,7 +93,7 @@ before applying the logarithm transformation: .. code:: python - {'MedInc': 0, 'HouseAge': 0} + {'MedInc': 0.0, 'HouseAge': 0.0} .. note:: @@ -298,6 +298,47 @@ And the constant values will be those from the dictionary: You can now apply `transform()` to transform all these variables. +With polars +----------- + +:class:`LogCpTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import LogCpTransformer + + df = pl.DataFrame({"var_1": [-2.0, -1.0, 0.0, 1.0, 2.0]}) + + tf = LogCpTransformer(variables=None) + tf.fit(df) + + print(tf.C_) + +.. code:: text + + {'var_1': 3.0} + +.. code:: python + + print(tf.transform(df)) + +.. code:: text + + shape: (5, 1) + ┌──────────┐ + │ var_1 │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.0 │ + │ 0.693147 │ + │ 1.098612 │ + │ 1.386294 │ + │ 1.609438 │ + └──────────┘ + + Additional resources -------------------- diff --git a/docs/user_guide/transformation/LogTransformer.rst b/docs/user_guide/transformation/LogTransformer.rst index b73d317d8..8bab8fd38 100644 --- a/docs/user_guide/transformation/LogTransformer.rst +++ b/docs/user_guide/transformation/LogTransformer.rst @@ -217,6 +217,48 @@ mapping each variable to its own constant (``C={"bmi": 2, "s3": 3}``), the same way you would with the deprecated :class:`LogCpTransformer()`. +With polars +----------- + +:class:`LogTransformer()` works in the same way with a polars dataframe, including +the ``C="auto"`` shift for variables that contain zero or negative values: + +.. code:: python + + import polars as pl + from feature_engine.transformation import LogTransformer + + df = pl.DataFrame({"var_1": [-2.0, -1.0, 0.0, 1.0, 2.0]}) + + logt = LogTransformer(variables=None, C="auto") + logt.fit(df) + + print(logt.C_) + +.. code:: text + + {'var_1': 3.0} + +.. code:: python + + print(logt.transform(df)) + +.. code:: text + + shape: (5, 1) + ┌──────────┐ + │ var_1 │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.0 │ + │ 0.693147 │ + │ 1.098612 │ + │ 1.386294 │ + │ 1.609438 │ + └──────────┘ + + Additional resources -------------------- diff --git a/docs/user_guide/transformation/PowerTransformer.rst b/docs/user_guide/transformation/PowerTransformer.rst index 7aef1c6d4..009fcfb68 100644 --- a/docs/user_guide/transformation/PowerTransformer.rst +++ b/docs/user_guide/transformation/PowerTransformer.rst @@ -423,6 +423,43 @@ Result of the inverse transformation: As we can see, the original data and the inverse transformed one are identical. +With polars +----------- + +:class:`PowerTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import PowerTransformer + + df = pl.DataFrame({ + "var_1": [4.0, 9.0, 16.0, 25.0, 100.0], + "var_2": [1.0, 8.0, 27.0, 64.0, 125.0], + }) + + tf = PowerTransformer(variables=None, exp=0.5) + tf.fit(df) + Xt = tf.transform(df) + + print(Xt) + +.. code:: text + + shape: (5, 2) + ┌───────┬──────────┐ + │ var_1 ┆ var_2 │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞═══════╪══════════╡ + │ 2.0 ┆ 1.0 │ + │ 3.0 ┆ 2.828427 │ + │ 4.0 ┆ 5.196152 │ + │ 5.0 ┆ 8.0 │ + │ 10.0 ┆ 11.18034 │ + └───────┴──────────┘ + + Considerations -------------- diff --git a/docs/user_guide/transformation/ReciprocalTransformer.rst b/docs/user_guide/transformation/ReciprocalTransformer.rst index e4aacf030..9f6336a12 100644 --- a/docs/user_guide/transformation/ReciprocalTransformer.rst +++ b/docs/user_guide/transformation/ReciprocalTransformer.rst @@ -227,6 +227,43 @@ symmetrically distributed across their value ranges: That's it! We've now applied different mathematical functions to stabilise the variance of the variables in the dataset. +With polars +----------- + +:class:`ReciprocalTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import ReciprocalTransformer + + df = pl.DataFrame({ + "ratio_1": [4.0, 5.0, 10.0, 20.0, 2.0], + "ratio_2": [0.5, 0.25, 0.2, 0.1, 1.0], + }) + + tf = ReciprocalTransformer(variables=None) + tf.fit(df) + Xt = tf.transform(df) + + print(Xt) + +.. code:: text + + shape: (5, 2) + ┌─────────┬─────────┐ + │ ratio_1 ┆ ratio_2 │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞═════════╪═════════╡ + │ 0.25 ┆ 2.0 │ + │ 0.2 ┆ 4.0 │ + │ 0.1 ┆ 5.0 │ + │ 0.05 ┆ 10.0 │ + │ 0.5 ┆ 1.0 │ + └─────────┴─────────┘ + + Alternatives to the reciprocal function --------------------------------------- diff --git a/docs/user_guide/transformation/YeoJohnsonTransformer.rst b/docs/user_guide/transformation/YeoJohnsonTransformer.rst index 160e6f795..7c6257356 100644 --- a/docs/user_guide/transformation/YeoJohnsonTransformer.rst +++ b/docs/user_guide/transformation/YeoJohnsonTransformer.rst @@ -201,6 +201,43 @@ values, using the `inverse_transform` method. test_unt = tf.inverse_transform(test_t) +With polars +----------- + +:class:`YeoJohnsonTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import YeoJohnsonTransformer + + df = pl.DataFrame({ + "var_1": [-4.0, -1.0, 0.0, 3.0, 10.0], + "var_2": [1.0, 8.0, 27.0, 64.0, 125.0], + }) + + tf = YeoJohnsonTransformer(variables=None) + tf.fit(df) + Xt = tf.transform(df) + + print(Xt) + +.. code:: text + + shape: (5, 2) + ┌───────────┬──────────┐ + │ var_1 ┆ var_2 │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞═══════════╪══════════╡ + │ -5.520511 ┆ 0.740576 │ + │ -1.129157 ┆ 2.723463 │ + │ 0.0 ┆ 4.64057 │ + │ 2.323426 ┆ 6.353727 │ + │ 6.13494 ┆ 7.905058 │ + └───────────┴──────────┘ + + Additional resources -------------------- diff --git a/docs/user_guide/variable_handling/check_all_variables.rst b/docs/user_guide/variable_handling/check_all_variables.rst index d35f7f53d..cb0e74e95 100644 --- a/docs/user_guide/variable_handling/check_all_variables.rst +++ b/docs/user_guide/variable_handling/check_all_variables.rst @@ -38,8 +38,8 @@ Now, we create the dataset: X["cat_var1"] = ["Hello"] * 1000 X["cat_var2"] = ["Bye"] * 1000 - X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="T") - X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="H") + X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="min") + X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="h") X["date3"] = ["2020-02-24"] * 1000 print(X.head()) @@ -89,4 +89,48 @@ Below we see the error message: .. code:: python - KeyError: 'Some of the variables are not in the dataframe.' \ No newline at end of file + KeyError: 'Some of the variables are not in the dataframe.' + +With polars +----------- + +:class:`check_all_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import check_all_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + + checked_vars = check_all_variables(X, ["num_var_1", "cat_var1", "date1"]) + + checked_vars + +The output is the list of variable names passed to the function: + +.. code:: python + + ['num_var_1', 'cat_var1', 'date1'] \ No newline at end of file diff --git a/docs/user_guide/variable_handling/check_categorical_variables.rst b/docs/user_guide/variable_handling/check_categorical_variables.rst index d311decd6..1aa610ec1 100644 --- a/docs/user_guide/variable_handling/check_categorical_variables.rst +++ b/docs/user_guide/variable_handling/check_categorical_variables.rst @@ -38,8 +38,8 @@ Now, we create the dataset: X["cat_var1"] = ["Hello"] * 1000 X["cat_var2"] = ["Bye"] * 1000 - X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="T") - X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="H") + X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="min") + X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="h") X["date3"] = ["2020-02-24"] * 1000 print(X.head()) @@ -89,4 +89,49 @@ Below we see the error message: .. code:: python TypeError: Some of the variables are not categorical. Please cast them as object - or categorical before using this transformer. \ No newline at end of file + or categorical before using this transformer. + +With polars +----------- + +:class:`check_categorical_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import check_categorical_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + + var_cat = check_categorical_variables(X, ["cat_var1", "date3"]) + + var_cat + +Both variables are of type string and hence, will be in the resulting list: + +.. code:: python + + ['cat_var1', 'date3'] + diff --git a/docs/user_guide/variable_handling/check_datetime_variables.rst b/docs/user_guide/variable_handling/check_datetime_variables.rst index e1fb86168..ab28a9eb4 100644 --- a/docs/user_guide/variable_handling/check_datetime_variables.rst +++ b/docs/user_guide/variable_handling/check_datetime_variables.rst @@ -38,8 +38,8 @@ Now, we create the dataset: X["cat_var1"] = ["Hello"] * 1000 X["cat_var2"] = ["Bye"] * 1000 - X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="T") - X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="H") + X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="min") + X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="h") X["date3"] = ["2020-02-24"] * 1000 print(X.head()) @@ -92,3 +92,49 @@ Below the error message: .. code:: python TypeError: Some of the variables are not or cannot be parsed as datetime. + +With polars +----------- + +:class:`check_datetime_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import check_datetime_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + + var_date = check_datetime_variables(X, ["date2", "date3"]) + + var_date + +In this case, both variables, if they can be parsed as datetime, will be in the +resulting list: + +.. code:: python + + ['date2', 'date3'] + diff --git a/docs/user_guide/variable_handling/check_numerical_variables.rst b/docs/user_guide/variable_handling/check_numerical_variables.rst index 795376850..b95255b5c 100644 --- a/docs/user_guide/variable_handling/check_numerical_variables.rst +++ b/docs/user_guide/variable_handling/check_numerical_variables.rst @@ -25,7 +25,7 @@ Now, we create the dataset: "City": ["London", "Manchester", "Liverpool", "Bristol"], "Age": [20, 21, 19, 18], "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), }) print(df.head()) @@ -67,3 +67,32 @@ Below we see the error message: TypeError: Some of the variables are not numerical. Please cast them as numerical before using this transformer. + +With polars +----------- + +:class:`check_numerical_variables()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from datetime import datetime + from feature_engine.variable_handling import check_numerical_variables + + df = pl.DataFrame({ + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": [datetime(2020, 2, 24, 0, i) for i in range(4)], + }) + + var_num = check_numerical_variables(df, ['Age', 'Marks']) + + var_num + +If the variables are numerical, the function returns their names in a list: + +.. code:: python + + ['Age', 'Marks'] diff --git a/docs/user_guide/variable_handling/find_all_variables.rst b/docs/user_guide/variable_handling/find_all_variables.rst index 055fac788..8c7835dd0 100644 --- a/docs/user_guide/variable_handling/find_all_variables.rst +++ b/docs/user_guide/variable_handling/find_all_variables.rst @@ -125,4 +125,68 @@ However, this command returns an empty list: X[[ 'date1', 'date2', 'date3']], exclude_datetime=True, return_empty=True, - ) \ No newline at end of file + ) + +With polars +----------- + +:class:`find_all_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import find_all_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + + vars_all = find_all_variables(X) + + vars_all + +We see the variable names in the list below: + +.. code:: python + + ['num_var_1', + 'num_var_2', + 'num_var_3', + 'num_var_4', + 'cat_var1', + 'cat_var2', + 'date1', + 'date2', + 'date3'] + +And, as with pandas, we can exclude the datetime variables: + +.. code:: python + + vars_all = find_all_variables(X, exclude_datetime=True) + + vars_all + +.. code:: python + + ['num_var_1', 'num_var_2', 'num_var_3', 'num_var_4', 'cat_var1', 'cat_var2'] \ No newline at end of file diff --git a/docs/user_guide/variable_handling/find_categorical_and_numerical_variables.rst b/docs/user_guide/variable_handling/find_categorical_and_numerical_variables.rst index 05d5807cc..0b27dc70a 100644 --- a/docs/user_guide/variable_handling/find_categorical_and_numerical_variables.rst +++ b/docs/user_guide/variable_handling/find_categorical_and_numerical_variables.rst @@ -126,4 +126,50 @@ To return empty lists instead, we set `return_empty` to `True`: find_categorical_and_numerical_variables( X[[ 'date1', 'date2', 'date3']], return_empty = True - ) \ No newline at end of file + ) + +With polars +----------- + +:class:`find_categorical_and_numerical_variables()` works in the same way with a +polars dataframe. Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import find_categorical_and_numerical_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + + var_cat, var_num = find_categorical_and_numerical_variables(X) + + var_cat, var_num + +Below we see the names of the categorical variables, followed by the names of the +numerical variables: + +.. code:: python + + (['cat_var1', 'cat_var2'], + ['num_var_1', 'num_var_2', 'num_var_3', 'num_var_4']) \ No newline at end of file diff --git a/docs/user_guide/variable_handling/find_categorical_variables.rst b/docs/user_guide/variable_handling/find_categorical_variables.rst index a00de72af..1ed6a23de 100644 --- a/docs/user_guide/variable_handling/find_categorical_variables.rst +++ b/docs/user_guide/variable_handling/find_categorical_variables.rst @@ -90,3 +90,52 @@ To return an empty list instead of the error we need to set `return_empty` to `T follows: `find_categorical_variables(X[colnames], return_empty=True)`. The previous command returns an empty list: `[]`. + +With polars +----------- + +:class:`find_categorical_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import find_categorical_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + +Now let's find the categorical variables: + +.. code:: python + + var_cat = find_categorical_variables(X) + + var_cat + +We see the variable names in the list below: + +.. code:: python + + ['cat_var1', 'cat_var2'] + diff --git a/docs/user_guide/variable_handling/find_datetime_variables.rst b/docs/user_guide/variable_handling/find_datetime_variables.rst index 443a43d4e..baa327873 100644 --- a/docs/user_guide/variable_handling/find_datetime_variables.rst +++ b/docs/user_guide/variable_handling/find_datetime_variables.rst @@ -87,3 +87,53 @@ can be parsed as datetime, it will be captured in the list as well. If there are no datetime variables, :class:`find_datetime_variables()` will raise an error. To return an empty list instead, use the argument `return_empty` to `True`. + +With polars +----------- + +:class:`find_datetime_variables()` works in the same way with a polars dataframe. +Let's create an equivalent toy dataset: + +.. code:: python + + import polars as pl + from datetime import datetime, timedelta + from sklearn.datasets import make_classification + from feature_engine.variable_handling import find_datetime_variables + + X, y = make_classification( + n_samples=1000, + n_features=4, + n_redundant=1, + n_clusters_per_class=1, + weights=[0.50], + class_sep=2, + random_state=1, + ) + + colnames = [f"num_var_{i+1}" for i in range(4)] + X = pl.DataFrame(X, schema=colnames) + + X = X.with_columns( + pl.lit("Hello").alias("cat_var1"), + pl.lit("Bye").alias("cat_var2"), + pl.Series("date1", [datetime(2020, 2, 24) + timedelta(minutes=i) for i in range(1000)]), + pl.Series("date2", [datetime(2021, 9, 29) + timedelta(hours=i) for i in range(1000)]), + pl.lit("2020-02-24").alias("date3"), + ) + +The dataframe has 3 datetime variables: two of them are native polars `Datetime` +columns, and one, `date3`, is an ISO-8601 string. Let's capture all 3: + +.. code:: python + + var_date = find_datetime_variables(X) + + var_date + +Below we see the variable names in the list: + +.. code:: python + + ['date1', 'date2', 'date3'] + diff --git a/docs/user_guide/variable_handling/find_numerical_variables.rst b/docs/user_guide/variable_handling/find_numerical_variables.rst index fcb4b40c4..aa315114d 100644 --- a/docs/user_guide/variable_handling/find_numerical_variables.rst +++ b/docs/user_guide/variable_handling/find_numerical_variables.rst @@ -68,3 +68,32 @@ need to set `return_empty` to `True`: find_numerical_variables(df[["Name", "City", "dob"]], return_empty=True) The previous command returns an empty list: `[]`. + +With polars +----------- + +:class:`find_numerical_variables()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from datetime import datetime + from feature_engine.variable_handling import find_numerical_variables + + df = pl.DataFrame({ + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": [datetime(2020, 2, 24, 0, i) for i in range(4)], + }) + + var_num = find_numerical_variables(df) + + var_num + +We see the names of the numerical variables in the list below: + +.. code:: python + + ['Age', 'Marks'] diff --git a/docs/user_guide/variable_handling/retain_variables_if_in_df.rst b/docs/user_guide/variable_handling/retain_variables_if_in_df.rst index 4f4ab71cd..c5f36ecf7 100644 --- a/docs/user_guide/variable_handling/retain_variables_if_in_df.rst +++ b/docs/user_guide/variable_handling/retain_variables_if_in_df.rst @@ -25,7 +25,7 @@ Now, we create the dataset: "City": ["London", "Manchester", "Liverpool", "Bristol"], "Age": [20, 21, 19, 18], "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), }) print(df.head()) @@ -59,6 +59,35 @@ We see the names of the subset of variables that are in the dataframe below: If none of variables in the list are in the dataset, :class:`retain_variables_if_in_df()` will raise an error. +With polars +----------- + +:class:`retain_variables_if_in_df()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from datetime import datetime + from feature_engine.variable_handling import retain_variables_if_in_df + + df = pl.DataFrame({ + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": [datetime(2020, 2, 24, 0, i) for i in range(4)], + }) + + vars_in_df = retain_variables_if_in_df(df, variables = ["Name", "City", "Dogs"]) + + vars_in_df + +We see the names of the subset of variables that are in the dataframe below: + +.. code:: python + + ['Name', 'City'] + Uses ---- diff --git a/feature_engine/_base_transformers/base_numerical.py b/feature_engine/_base_transformers/base_numerical.py index fed663213..3cf78254a 100644 --- a/feature_engine/_base_transformers/base_numerical.py +++ b/feature_engine/_base_transformers/base_numerical.py @@ -3,7 +3,9 @@ shared by most transformers, like checking that input is a df, the size, NA, etc. """ -import pandas as pd +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted @@ -28,7 +30,7 @@ class BaseNumericalTransformer( variable transformers, discretisers, math combination. """ - def _fit_setup(self, X: pd.DataFrame): + def _fit_setup(self, X: IntoDataFrame): """ Checks that input is a dataframe, finds numerical variables, or alternatively checks that variables entered by the user are of type numerical, and checks @@ -38,12 +40,12 @@ def _fit_setup(self, X: pd.DataFrame): Parameters ---------- - X : Pandas DataFrame + X : dataframe Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array + If the input is not a recognised dataframe If any of the user provided variables are not numerical ValueError If there are no numerical variables in the df or the df is empty @@ -51,7 +53,7 @@ def _fit_setup(self, X: pd.DataFrame): Returns ------- - X : Pandas DataFrame + X : dataframe The same dataframe entered as parameter variables_ : List @@ -77,31 +79,34 @@ def _get_feature_names_in(self, X): """Get the names and number of features in the train set (the dataframe used during fit).""" - self.feature_names_in_ = X.columns.tolist() + if nwd.is_pandas_dataframe(X) is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw.from_native(X, eager_only=True).columns self.n_features_in_ = X.shape[1] return self - def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: + def _check_transform_input_and_state(self, X: IntoDataFrame) -> IntoDataFrame: """ Checks that the input is a dataframe and of the same size than the one used in the fit() method. Checks absence of NA and Inf. Parameters ---------- - X : Pandas DataFrame + X : dataframe Raises ------ TypeError - If the input is not a Pandas DataFrame + If the input is not a recognised dataframe ValueError - If the variable(s) contain null values - If the df has different number of features than the df used in fit() Returns ------- - X : Pandas DataFrame. + X : dataframe. The same dataframe entered by the user. """ @@ -119,7 +124,12 @@ def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: _check_contains_inf(X, self.variables_) # reorder variables to match train set - X = X[self.feature_names_in_] + if nwd.is_pandas_dataframe(X) is True: + X = X[self.feature_names_in_] + else: + X = nw.from_native(X, eager_only=True).select( + self.feature_names_in_ + ).to_native() return X diff --git a/feature_engine/_base_transformers/mixins.py b/feature_engine/_base_transformers/mixins.py index 9207873be..6517f9207 100644 --- a/feature_engine/_base_transformers/mixins.py +++ b/feature_engine/_base_transformers/mixins.py @@ -1,6 +1,8 @@ from typing import Dict, List, Tuple, Union -import pandas as pd +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame, IntoSeries from numpy import ndarray from numpy.typing import ArrayLike from sklearn.utils.validation import check_is_fitted @@ -15,7 +17,7 @@ class TransformXyMixin: - def transform_x_y(self, X: pd.DataFrame, y: pd.Series): + def transform_x_y(self, X: IntoDataFrame, y: IntoSeries): """ Transform, align and adjust both X and y based on the transformations applied to X, ensuring that they correspond to the same set of rows if any were @@ -23,32 +25,46 @@ def transform_x_y(self, X: pd.DataFrame, y: pd.Series): Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The dataframe to transform. - y: pandas Series or Dataframe of length = n_samples + y: Series or Dataframe of length = n_samples The target variable to transform. Can be multi-output. Returns ------- - X_new: pandas dataframe + X_new: dataframe The transformed dataframe of shape [n_samples - n_rows, n_features]. It may contain less rows than the original dataset. - y_new: pandas Series or DataFrame + y_new: Series or DataFrame The transformed target variable of length [n_samples - n_rows]. It contains as many rows as those left in X_new. """ X, y = check_X_y(X, y) - X = self.transform(X) - y = y.loc[X.index] + + if nwd.is_pandas_dataframe(X) is True: + X = self.transform(X) + y = y.loc[X.index] + else: + row_index_col = "__feature_engine_row_index__" + nw_X = nw.from_native(X, eager_only=True).with_row_index(row_index_col) + X = self.transform(nw_X.to_native()) + nw_X = nw.from_native(X, eager_only=True) + row_positions = nw_X.get_column(row_index_col) + X = nw_X.drop(row_index_col).to_native() + if nwd.is_into_series(y): + y = nw.from_native(y, series_only=True)[row_positions].to_native() + else: + y = nw.from_native(y, eager_only=True)[row_positions].to_native() + return X, y class FitFromDictMixin: def _fit_from_dict( - self, X: pd.DataFrame, user_dict_: Dict - ) -> Tuple[pd.DataFrame, List[Union[str, int]]]: + self, X: IntoDataFrame, user_dict_: Dict + ) -> Tuple[IntoDataFrame, List[Union[str, int]]]: """ Checks that input is a dataframe, checks that variables in the dictionary entered by the user are of type numerical. Does not assign any @@ -57,7 +73,7 @@ def _fit_from_dict( Parameters ---------- - X : Pandas DataFrame + X : dataframe user_dict_ : Dictionary. Default = None Any dictionary allowed by the transformer and entered by user. @@ -65,7 +81,7 @@ def _fit_from_dict( Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array + If the input is not a recognised dataframe If any of the variables in the dictionary are not numerical ValueError If there are no numerical variables in the df or the df is empty @@ -73,7 +89,7 @@ def _fit_from_dict( Returns ------- - X : Pandas DataFrame + X : dataframe The same dataframe entered as parameter variables_ : List @@ -118,47 +134,20 @@ def get_feature_names_out( check_is_fitted(self) if input_features is not None: - # If input to fit is an array, then the variable names in - # feature_names_in_ are "x0", "x1","x2" ..."xn". - if self.feature_names_in_ == [f"x{i}" for i in range(self.n_features_in_)]: - - # If the input was an array, we let the user enter the variable names. - if len(input_features) == self.n_features_in_: - if isinstance(input_features, list): - feature_names = input_features - else: - feature_names = list(input_features) - - # For transformers that add features to the data. - feature_names = self._add_new_feature_names(feature_names) - - # For transformers that remove features from data, i..e, selectors. - feature_names = self._remove_feature_names( - feature_names, indices=True - ) - - return feature_names - - else: - raise ValueError( - "The number of input_features does not match the number of " - "features seen in the dataframe used in fit." - ) + msg = "input_features is not equal to feature_names_in_" + if isinstance(input_features, list): + if input_features != self.feature_names_in_: + raise ValueError(msg) + elif isinstance(input_features, ndarray) or ( + nwd.is_pandas_index(input_features) is True + ): + if list(input_features) != self.feature_names_in_: + raise ValueError(msg) else: - msg = "input_features is not equal to feature_names_in_" - if isinstance(input_features, list): - if input_features != self.feature_names_in_: - raise ValueError(msg) - elif isinstance(input_features, ndarray) or isinstance( - input_features, pd.core.indexes.base.Index - ): - if list(input_features) != self.feature_names_in_: - raise ValueError(msg) - else: - raise ValueError( - "input_features must be a list or an array. " - "Got {input_features} instead." - ) + raise ValueError( + "input_features must be a list or an array. " + "Got {input_features} instead." + ) feature_names = self.feature_names_in_ @@ -166,7 +155,7 @@ def get_feature_names_out( feature_names = self._add_new_feature_names(feature_names) # For transformers that remove features from data, i..e, selectors. - feature_names = self._remove_feature_names(feature_names, indices=False) + feature_names = self._remove_feature_names(feature_names) return feature_names @@ -183,14 +172,10 @@ def _add_new_feature_names(self, feature_names): return feature_names - def _remove_feature_names(self, feature_names, indices=False) -> List: + def _remove_feature_names(self, feature_names) -> List: # For transformers that remove features from data, i..e, selectors. if hasattr(self, "features_to_drop_"): - if indices is True: - mask = self.get_support(indices=True) - feature_names = [feature_names[i] for i in mask] - else: - feature_names = [ - f for f in feature_names if f not in self.features_to_drop_ - ] + feature_names = [ + f for f in feature_names if f not in self.features_to_drop_ + ] return feature_names diff --git a/feature_engine/creation/base_creation.py b/feature_engine/creation/base_creation.py index c294045f4..1c17bb647 100644 --- a/feature_engine/creation/base_creation.py +++ b/feature_engine/creation/base_creation.py @@ -1,6 +1,8 @@ from typing import Optional -import pandas as pd +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted @@ -37,16 +39,16 @@ def __init__( self.missing_values = missing_values self.drop_original = drop_original - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn parameters. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. - y: pandas Series, or np.array. Defaults to None. + y: Series, or np.array. Defaults to None. It is not needed in this transformer. You can pass y or None. """ @@ -71,25 +73,28 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): _check_contains_inf(X, self.reference) # save input features - self.feature_names_in_ = X.columns.tolist() + if nwd.is_pandas_dataframe(X) is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw.from_native(X, eager_only=True).columns # save train set shape self.n_features_in_ = X.shape[1] return self - def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: + def _check_transform_input_and_state(self, X: IntoDataFrame) -> IntoDataFrame: """ Common input and transformer checks. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: Pandas dataframe + X_new: dataframe The dataframe with the original variables plus the new variables. """ @@ -111,7 +116,12 @@ def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: _check_contains_inf(X, self.reference) # reorder variables to match train set - X = X[self.feature_names_in_] + if nwd.is_pandas_dataframe(X) is True: + X = X[self.feature_names_in_] + else: + X = nw.from_native(X, eager_only=True).select( + self.feature_names_in_ + ).to_native() return X diff --git a/feature_engine/creation/cyclical_features.py b/feature_engine/creation/cyclical_features.py index 24018b0cd..bcae83299 100644 --- a/feature_engine/creation/cyclical_features.py +++ b/feature_engine/creation/cyclical_features.py @@ -1,7 +1,8 @@ from typing import Dict, List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._base_transformers.mixins import ( @@ -122,6 +123,30 @@ class CyclicalFeatures( 5 2 1.224647e-16 -1.000000e+00 6 1 1.000000e+00 6.123234e-17 7 2 1.224647e-16 -1.000000e+00 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import CyclicalFeatures + >>> X = pl.DataFrame({"x": [1, 4, 3, 3, 4, 2, 1, 2]}) + >>> cf = CyclicalFeatures() + >>> cf.fit(X) + >>> cf.transform(X) + shape: (8, 3) + ┌─────┬─────────────┬─────────────┐ + │ x ┆ x_sin ┆ x_cos │ + │ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ f64 │ + ╞═════╪═════════════╪═════════════╡ + │ 1 ┆ 1.0 ┆ 6.1232e-17 │ + │ 4 ┆ -2.4493e-16 ┆ 1.0 │ + │ 3 ┆ -1.0 ┆ -1.8370e-16 │ + │ 3 ┆ -1.0 ┆ -1.8370e-16 │ + │ 4 ┆ -2.4493e-16 ┆ 1.0 │ + │ 2 ┆ 1.2246e-16 ┆ -1.0 │ + │ 1 ┆ 1.0 ┆ 6.1232e-17 │ + │ 2 ┆ 1.2246e-16 ┆ -1.0 │ + └─────┴─────────────┴─────────────┘ """ def __init__( @@ -141,22 +166,36 @@ def __init__( self.max_values = max_values self.drop_original = drop_original - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learns the maximum value of each variable. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ if self.max_values is None: X, variables_ = self._fit_setup(X) - max_values_ = X[variables_].max().to_dict() + if len(variables_) == 0: + # return_empty=True can leave variables_ empty; narwhals' + # select([]) collapses row count too, so .to_numpy().max() + # would fail on a genuinely empty selection. + max_values_ = {} + else: + max_arr = ( + nw.from_native(X, eager_only=True) + .select(variables_) + .to_numpy() + .max(axis=0) + ) + # .tolist() converts numpy scalars to plain Python int/float, + # matching the dtype .to_dict() used to return. + max_values_ = dict(zip(variables_, max_arr.tolist())) else: X, variables_ = super()._fit_from_dict(X, self.max_values) max_values_ = self.max_values @@ -167,29 +206,31 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame): + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Creates new features using the cyclical transformations. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe. + X_new: dataframe. The original dataframe plus the additional features. """ X = self._check_transform_input_and_state(X) + new_cols = [] for variable in self.variables_: - max_value = self.max_values_[variable] - X[f"{variable}_sin"] = np.sin(X[variable] * (2.0 * np.pi / max_value)) - X[f"{variable}_cos"] = np.cos(X[variable] * (2.0 * np.pi / max_value)) - - if self.drop_original: - X.drop(columns=self.variables_, inplace=True) + scaled = nw.col(variable) * (2.0 * np.pi / self.max_values_[variable]) + new_cols.append(scaled.sin().alias(f"{variable}_sin")) + new_cols.append(scaled.cos().alias(f"{variable}_cos")) + nw_X = nw.from_native(X, eager_only=True).with_columns(*new_cols) + if self.drop_original is True: + nw_X = nw_X.drop(self.variables_) + X = nw_X.to_native() return X diff --git a/feature_engine/creation/decision_tree_features.py b/feature_engine/creation/decision_tree_features.py index aa76d4bbd..135a62815 100644 --- a/feature_engine/creation/decision_tree_features.py +++ b/feature_engine/creation/decision_tree_features.py @@ -1,8 +1,11 @@ import itertools from typing import Any, Dict, Iterable, List, Optional, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np -import pandas as pd +from joblib import Parallel, delayed +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.base import BaseEstimator, TransformerMixin from sklearn.model_selection import GridSearchCV from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor @@ -131,6 +134,15 @@ class DecisionTreeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMi DecisionTreeClassifier(). For reproducibility it is recommended to set the random_state to an integer. + n_jobs: int, default=None + The number of jobs to run in parallel when training the decision trees + across feature combinations. Trees are fit using threads rather than + processes, since fitting a decision tree releases the GIL for the bulk + of its computation, which avoids the overhead of copying the entire + dataframe to separate worker processes. `None` means 1, i.e. sequential + training (this transformer's original behaviour); `-1` means using all + available processors. + {missing_values} {drop_original} @@ -210,6 +222,36 @@ class DecisionTreeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMi 7 4.24 8 4.24 9 6.00 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import DecisionTreeFeatures + >>> X = pl.DataFrame({ + ... "Age": [20, 44, 19, 33, 51, 40, 41, 37, 30, 54], + ... "Height": [164, 150, 178, 158, 188, 190, 168, 174, 176, 171], + ... }) + >>> y = [4.1, 5.8, 3.9, 6.2, 4.3, 4.5, 7.2, 4.4, 4.1, 6.7] + >>> dtf = DecisionTreeFeatures(features_to_combine=1) + >>> dtf.fit(X, y) + >>> dtf.transform(X) + shape: (10, 4) + ┌─────┬────────┬───────────┬──────────────┐ + │ Age ┆ Height ┆ tree(Age) ┆ tree(Height) │ + │ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ i64 ┆ f64 ┆ f64 │ + ╞═════╪════════╪═══════════╪══════════════╡ + │ 20 ┆ 164 ┆ 4.533333 ┆ 5.366667 │ + │ 44 ┆ 150 ┆ 6.0 ┆ 5.366667 │ + │ 19 ┆ 178 ┆ 4.533333 ┆ 4.133333 │ + │ 33 ┆ 158 ┆ 4.533333 ┆ 5.366667 │ + │ 51 ┆ 188 ┆ 6.0 ┆ 4.4 │ + │ 40 ┆ 190 ┆ 4.533333 ┆ 4.4 │ + │ 41 ┆ 168 ┆ 6.0 ┆ 6.95 │ + │ 37 ┆ 174 ┆ 4.533333 ┆ 4.133333 │ + │ 30 ┆ 176 ┆ 4.533333 ┆ 4.133333 │ + │ 54 ┆ 171 ┆ 6.0 ┆ 6.95 │ + └─────┴────────┴───────────┴──────────────┘ """ def __init__( @@ -223,6 +265,7 @@ def __init__( param_grid: Optional[Dict[str, Union[str, int, float, List[int]]]] = None, regression: bool = True, random_state: int = 0, + n_jobs: Optional[int] = None, missing_values: str = "raise", drop_original: bool = False, ) -> None: @@ -251,21 +294,22 @@ def __init__( self.param_grid = param_grid self.regression = regression self.random_state = random_state + self.n_jobs = n_jobs self.missing_values = missing_values self.drop_original = drop_original - def fit(self, X: pd.DataFrame, y: pd.Series): + def fit(self, X: IntoDataFrame, y: IntoSeries): """ Fits decision trees based on the input variable combinations with cross-validation and grid-search for hyperparameters. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series or np.array = [n_samples,] + y: Series or np.array = [n_samples,] The target variable that is used to train the decision tree. """ # confirm model type and target variables are compatible. @@ -302,39 +346,48 @@ def fit(self, X: pd.DataFrame, y: pd.Series): how_to_combine=self.features_to_combine, variables=variables_ ) - estimators_ = [] - for features in input_features: - estimator = self._make_decision_tree(param_grid=param_grid) + is_pandas = nwd.is_pandas_dataframe(X) + nw_X = nw.from_native(X, eager_only=True) + X_subs = [] + for features in input_features: # single feature models - if isinstance(features, str): - estimator.fit(X[features].to_frame(), y) + if isinstance(features, (str, int)): + X_sub = nw_X.get_column(features).to_frame().to_native() # multi feature models + elif is_pandas is True: + X_sub = X[features] else: - estimator.fit(X[features], y) + X_sub = nw_X.select(features).to_native() + X_subs.append(X_sub) - estimators_.append(estimator) + estimators_ = Parallel(n_jobs=self.n_jobs, prefer="threads")( + delayed(self._fit_one_tree)(X_sub, y, param_grid) for X_sub in X_subs + ) self.variables_ = variables_ self.input_features_ = input_features self.estimators_ = estimators_ - self.feature_names_in_ = X.columns.tolist() + if is_pandas is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw_X.columns self.n_features_in_ = X.shape[1] return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Create and add new variables. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe. + X_new: dataframe. Either the original dataframe plus the new features or a dataframe of only the new features. """ @@ -351,50 +404,64 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: _check_contains_na(X, self.variables_) _check_contains_inf(X, self.variables_) - # reorder variables to match train set - X = X[self.feature_names_in_] - - # create new features and add them to the original dataframe - # if regression or multiclass, we return the output of predict() - if self.regression is True: - for features, estimator in zip(self.input_features_, self.estimators_): - if isinstance(features, str): - preds = estimator.predict(X[features].to_frame()) - if self.precision is not None: - preds = np.round(preds, self.precision) - X.loc[:, f"tree({features})"] = preds - else: - preds = estimator.predict(X[features]) - if self.precision is not None: - preds = np.round(preds, self.precision) - X.loc[:, f"tree({features})"] = preds + is_pandas = nwd.is_pandas_dataframe(X) + # reorder variables to match train set + if is_pandas is True: + X = X[self.feature_names_in_] + else: + X = ( + nw.from_native(X, eager_only=True) + .select(self.feature_names_in_) + .to_native() + ) + nw_X = nw.from_native(X, eager_only=True) + + def get_x_sub(features): + if isinstance(features, (str, int)): + return nw_X.get_column(features).to_frame().to_native() + if is_pandas is True: + return X[features] + return nw_X.select(features).to_native() + + new_series = [] + new_columns = {} + # if regression or multiclass, we return the output of predict(); # if binary classification, we return the probability - elif self._is_binary == "binary": - for features, estimator in zip(self.input_features_, self.estimators_): - if isinstance(features, str): - preds = estimator.predict_proba(X[features].to_frame()) - if self.precision is not None: - preds = np.round(preds, self.precision) - X.loc[:, f"tree({features})"] = preds[:, 1] - else: - preds = estimator.predict_proba(X[features]) - if self.precision is not None: - preds = np.round(preds, self.precision) - X.loc[:, f"tree({features})"] = preds[:, 1] + for features, estimator in zip(self.input_features_, self.estimators_): + X_sub = get_x_sub(features) + col_name = f"tree({features})" + + if self.regression is True: + preds = estimator.predict(X_sub) + if self.precision is not None: + preds = np.round(preds, self.precision) + elif self._is_binary == "binary": + preds = estimator.predict_proba(X_sub)[:, 1] + if self.precision is not None: + preds = np.round(preds, self.precision) + else: + preds = estimator.predict(X_sub) - # if multiclass, we return the output of predict() - else: - for features, estimator in zip(self.input_features_, self.estimators_): - if isinstance(features, str): - preds = estimator.predict(X[features].to_frame()) - X.loc[:, f"tree({features})"] = preds - else: - preds = estimator.predict(X[features]) - X.loc[:, f"tree({features})"] = preds + if is_pandas is True: + new_columns[col_name] = preds + else: + new_series.append( + nw.new_series(col_name, preds, backend=nw_X.implementation) + ) - if self.drop_original: - X.drop(columns=self.variables_, inplace=True) + if is_pandas is True: + # assign() still inserts columns one at a time internally, so it + # doesn't avoid fragmentation with many feature combinations; + # building one DataFrame and joining it does (single insertion). + X = X.join(type(X)(new_columns, index=X.index)) + if self.drop_original is True: + X = X.drop(columns=self.variables_) + else: + nw_X = nw.from_native(X, eager_only=True).with_columns(*new_series) + if self.drop_original is True: + nw_X = nw_X.drop(self.variables_) + X = nw_X.to_native() return X @@ -414,6 +481,12 @@ def _make_decision_tree(self, param_grid: Dict): return tree_model + def _fit_one_tree(self, X_sub: IntoDataFrame, y: IntoSeries, param_grid: Dict): + """Instantiate and fit one decision tree on one feature combination.""" + estimator = self._make_decision_tree(param_grid=param_grid) + estimator.fit(X_sub, y) + return estimator + def _create_variable_combinations( self, variables: List, diff --git a/feature_engine/creation/geo_features.py b/feature_engine/creation/geo_features.py index bb2698d07..c82660d32 100644 --- a/feature_engine/creation/geo_features.py +++ b/feature_engine/creation/geo_features.py @@ -3,8 +3,10 @@ from typing import List, Literal, Optional, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted @@ -142,10 +144,39 @@ class GeoDistanceFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMix >>> gdt.fit(X) >>> X = gdt.transform(X) >>> X - origin_lat origin_lon dest_lat dest_lon geo_distance - 0 40.7128 -74.0060 34.0522 -118.2437 3935.746254 - 1 34.0522 -118.2437 41.8781 -87.6298 2808.517344 - 2 41.8781 -87.6298 40.7128 -74.0060 1144.286561 + origin_lat origin_lon dest_lat dest_lon geo_distance + 0 40.7128 -74.0060 34.0522 -118.2437 3935.746255 + 1 34.0522 -118.2437 41.8781 -87.6298 2803.971507 + 2 41.8781 -87.6298 40.7128 -74.0060 1144.291274 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import GeoDistanceFeatures + >>> X = pl.DataFrame({ + ... "origin_lat": [40.7128, 34.0522, 41.8781], + ... "origin_lon": [-74.0060, -118.2437, -87.6298], + ... "dest_lat": [34.0522, 41.8781, 40.7128], + ... "dest_lon": [-118.2437, -87.6298, -74.0060], + ... }) + >>> gdt = GeoDistanceFeatures( + ... lat1="origin_lat", lon1="origin_lon", + ... lat2="dest_lat", lon2="dest_lon", + ... method="haversine", output_unit="km" + ... ) + >>> gdt.fit(X) + >>> X = gdt.transform(X) + >>> X + shape: (3, 5) + ┌────────────┬────────────┬──────────┬───────────┬──────────────┐ + │ origin_lat ┆ origin_lon ┆ dest_lat ┆ dest_lon ┆ geo_distance │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ + ╞════════════╪════════════╪══════════╪═══════════╪══════════════╡ + │ 40.7128 ┆ -74.006 ┆ 34.0522 ┆ -118.2437 ┆ 3935.746255 │ + │ 34.0522 ┆ -118.2437 ┆ 41.8781 ┆ -87.6298 ┆ 2803.971507 │ + │ 41.8781 ┆ -87.6298 ┆ 40.7128 ┆ -74.006 ┆ 1144.291274 │ + └────────────┴────────────┴──────────┴───────────┴──────────────┘ """ def __init__( @@ -213,16 +244,16 @@ def __init__( self.drop_original = drop_original self.validate_ranges = validate_ranges - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn parameters. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. - y: pandas Series, or np.array. Defaults to None. + y: Series, or np.array. Defaults to None. It is not needed in this transformer. You can pass y or None. Returns @@ -233,6 +264,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check input dataframe X = check_X(X) + is_pandas = nwd.is_pandas_dataframe(X) # Coordinate variables variables: List[Union[str, int]] = [ @@ -243,7 +275,11 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): ] # Check all coordinate columns exist - missing = set(variables) - set(X.columns) + if is_pandas is True: + columns = set(X.columns) + else: + columns = set(nw.from_native(X, eager_only=True).columns) + missing = set(variables) - columns if missing: raise ValueError( f"Coordinate columns {missing} are not present in the dataframe." @@ -256,42 +292,61 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): _check_contains_na(X, variables) # Validate coordinate ranges if enabled - if self.validate_ranges: - for lat_col in [self.lat1, self.lat2]: - if (X[lat_col].abs() > 90).any(): - raise ValueError( - f"Latitude values in '{lat_col}' must be between -90 and 90." - ) - - for lon_col in [self.lon1, self.lon2]: - if (X[lon_col].abs() > 180).any(): - raise ValueError( - f"Longitude values in '{lon_col}' must be between -180 and 180." - ) + if self.validate_ranges is True: + self._validate_coordinate_ranges(X, is_pandas) # save coordinate variables self.variables_ = variables # save input features - self.feature_names_in_ = X.columns.tolist() + if is_pandas is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw.from_native(X, eager_only=True).columns # save train set shape self.n_features_in_ = X.shape[1] return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def _validate_coordinate_ranges(self, X: IntoDataFrame, is_pandas: bool) -> None: + """Raise if any latitude/longitude value falls outside its valid range.""" + if is_pandas is True: + for lat_col in [self.lat1, self.lat2]: + if (X[lat_col].abs() > 90).any(): + raise ValueError( + f"Latitude values in '{lat_col}' must be between -90 and 90." + ) + for lon_col in [self.lon1, self.lon2]: + if (X[lon_col].abs() > 180).any(): + raise ValueError( + f"Longitude values in '{lon_col}' must be between -180 and 180." + ) + else: + nw_X = nw.from_native(X, eager_only=True) + for lat_col in [self.lat1, self.lat2]: + if nw_X.select((nw.col(lat_col).abs() > 90).any()).to_numpy().any(): + raise ValueError( + f"Latitude values in '{lat_col}' must be between -90 and 90." + ) + for lon_col in [self.lon1, self.lon2]: + if nw_X.select((nw.col(lon_col).abs() > 180).any()).to_numpy().any(): + raise ValueError( + f"Longitude values in '{lon_col}' must be between -180 and 180." + ) + + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Calculate distances and add them as a new column. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the new distance column added. """ @@ -307,36 +362,43 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Check for missing values _check_contains_na(X, self.variables_) - # reorder variables to match train set - X = X[self.feature_names_in_] + is_pandas = nwd.is_pandas_dataframe(X) is True + + # reorder variables to match train set, and extract coordinate arrays + if is_pandas is True: + X = X[self.feature_names_in_] + lat1 = X[self.lat1].to_numpy() + lon1 = X[self.lon1].to_numpy() + lat2 = X[self.lat2].to_numpy() + lon2 = X[self.lon2].to_numpy() + else: + nw_X = nw.from_native(X, eager_only=True).select(self.feature_names_in_) + lat1 = nw_X.get_column(self.lat1).to_numpy() + lon1 = nw_X.get_column(self.lon1).to_numpy() + lat2 = nw_X.get_column(self.lat2).to_numpy() + lon2 = nw_X.get_column(self.lon2).to_numpy() # Calculate distance based on method if self.method == "haversine": - distances = self._haversine_distance( - X[self.lat1].values, - X[self.lon1].values, - X[self.lat2].values, - X[self.lon2].values, - ) + distances = self._haversine_distance(lat1, lon1, lat2, lon2) elif self.method == "euclidean": - distances = self._euclidean_distance( - X[self.lat1].values, - X[self.lon1].values, - X[self.lat2].values, - X[self.lon2].values, - ) + distances = self._euclidean_distance(lat1, lon1, lat2, lon2) else: # manhattan - distances = self._manhattan_distance( - X[self.lat1].values, - X[self.lon1].values, - X[self.lat2].values, - X[self.lon2].values, - ) - - X[self.output_col] = distances + distances = self._manhattan_distance(lat1, lon1, lat2, lon2) - if self.drop_original: - X = X.drop(columns=self.variables_) + if is_pandas is True: + X[self.output_col] = distances + if self.drop_original is True: + X = X.drop(columns=self.variables_) + else: + nw_X = nw_X.with_columns( + nw.new_series( + self.output_col, distances, backend=nw_X.implementation + ) + ) + if self.drop_original is True: + nw_X = nw_X.drop(self.variables_) + X = nw_X.to_native() return X diff --git a/feature_engine/creation/math_features.py b/feature_engine/creation/math_features.py index bea520bb1..edd36bca4 100644 --- a/feature_engine/creation/math_features.py +++ b/feature_engine/creation/math_features.py @@ -1,8 +1,10 @@ import warnings from typing import Any, List, Optional, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, @@ -21,7 +23,10 @@ from feature_engine._docstrings.substitute import Substitution from feature_engine.creation.base_creation import BaseCreation -_PANDAS_LT_3 = int(pd.__version__.split(".")[0]) < 3 + +def _pandas_version() -> int: + return int(nwd.get_pandas().__version__.split(".")[0]) + # In pandas < 3, agg() maps these callables to the pandas methods and warns that # this will change; the string alias keeps that behaviour (e.g., np.std -> @@ -83,7 +88,11 @@ class MathFeatures(BaseCreation): """ MathFeatures() applies functions across multiple features returning one or more additional features as a result. Common reductions use vectorized NumPy - operations. Other functions fall back to `pandas.agg()` with `axis=1`. + operations. Other functions fall back to `pandas.agg()` with `axis=1` for + pandas input, or to polars' native `map_rows()` for polars input — in that + case, the callable receives each row as a plain tuple, not a `Series`, so + it must not rely on `Series` methods (e.g. use `max(row)` instead of + `row.max()`) to work on both backends. For supported aggregation functions, see `pandas documentation `_. @@ -174,11 +183,30 @@ class MathFeatures(BaseCreation): >>> mf = MathFeatures(variables = ["x1","x2"], func = "mean") >>> mf.fit(X) - >>> mf.transform(X)) + >>> mf.transform(X) x1 x2 mean_x1_x2 0 1 4 2.5 1 2 5 3.5 2 3 6 4.5 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import MathFeatures + >>> X = pl.DataFrame({"x1": [1, 2, 3], "x2": [4, 5, 6]}) + >>> mf = MathFeatures(variables=["x1", "x2"], func="sum") + >>> mf.fit(X) + >>> mf.transform(X) + shape: (3, 3) + ┌─────┬─────┬───────────┐ + │ x1 ┆ x2 ┆ sum_x1_x2 │ + │ --- ┆ --- ┆ --- │ + │ i64 ┆ i64 ┆ i64 │ + ╞═════╪═════╪═══════════╡ + │ 1 ┆ 4 ┆ 5 │ + │ 2 ┆ 5 ┆ 7 │ + │ 3 ┆ 6 ┆ 9 │ + └─────┴─────┴───────────┘ """ def __init__( @@ -237,18 +265,18 @@ def __init__( self.func = func self.new_variables_names = new_variables_names - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Create and add new variables. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe, shape = [n_samples, n_features + n_operations] + X_new: dataframe, shape = [n_samples, n_features + n_operations] The input dataframe plus the new variables. """ X = self._check_transform_input_and_state(X) @@ -256,41 +284,72 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: new_variable_names = self._get_new_features_name() func = self.func - if _PANDAS_LT_3: + is_pandas = nwd.is_pandas_dataframe(X) + if is_pandas is True and _pandas_version() < 3: if isinstance(func, list): func = [_FUNC_TO_STRING_ALIAS.get(fun, fun) for fun in func] else: func = _FUNC_TO_STRING_ALIAS.get(func, func) - variables = X[self.variables] functions = func if isinstance(func, list) else [func] reducers = [_get_numpy_reducer(fun) for fun in functions] - values = variables.to_numpy() + + nw_X = nw.from_native(X, eager_only=True) + if is_pandas is True: + values = X[self.variables].to_numpy() + else: + values = nw_X.select(self.variables).to_numpy() # Nullable extension dtypes produce object arrays. Keep those, custom - # callables, and less common pandas aggregations on the exact legacy path. + # callables, and less common aggregations on the fallback path below. if reducers and values.dtype.kind in "biuf" and all(reducers): - results = [] - for reducer, kwargs in reducers: + new_series = [] + for (reducer, kwargs), name in zip(reducers, new_variable_names): # pandas' named reductions do not warn for empty/all-missing rows. # NumPy returns the same values but emits RuntimeWarning for some # reducers, so silence only those warnings on this equivalent path. with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) result = reducer(values, axis=1, **kwargs) - results.append(pd.Series(result, index=X.index)) - - result = results[0] if len(results) == 1 else pd.concat(results, axis=1) - else: - result = variables.agg(func, axis=1) - - if len(new_variable_names) == 1: - X[new_variable_names[0]] = result + new_series.append( + nw.new_series(name, result, backend=nw_X.implementation) + ) + nw_X = nw_X.with_columns(*new_series) + if self.drop_original is True: + nw_X = nw_X.drop(self.variables) + X = nw_X.to_native() + elif is_pandas is True: + result = X[self.variables].agg(func, axis=1) + if len(new_variable_names) == 1: + X[new_variable_names[0]] = result + else: + X[new_variable_names] = result + if self.drop_original is True: + X = X.drop(columns=self.variables) else: - X[new_variable_names] = result - - if self.drop_original: - X.drop(columns=self.variables, inplace=True) + # polars has no equivalent to pandas' agg(func, axis=1): apply each + # function natively via map_rows, one call per function. map_rows + # passes each row as a plain tuple, not a Series, so callables that + # rely on Series methods (e.g. `row.max()`) need `max(row)` instead. + sub_native = nw_X.select(self.variables).to_native() + new_series = [] + for fun, name in zip(functions, new_variable_names): + if not callable(fun): + raise NotImplementedError( + f"'{fun}' has no NumPy-vectorized implementation, and " + "non-callable aggregation names are not supported for " + "polars input. Pass a Python callable instead." + ) + result_df = sub_native.map_rows(fun) + new_series.append( + nw.new_series( + name, result_df.to_series(0), backend=nw_X.implementation + ) + ) + nw_X = nw_X.with_columns(*new_series) + if self.drop_original is True: + nw_X = nw_X.drop(self.variables) + X = nw_X.to_native() return X diff --git a/feature_engine/creation/relative_features.py b/feature_engine/creation/relative_features.py index 5b2957bff..d664c2448 100644 --- a/feature_engine/creation/relative_features.py +++ b/feature_engine/creation/relative_features.py @@ -1,6 +1,8 @@ from typing import List, Union -import pandas as pd +import narwhals as nw +import numpy as np +from narwhals.typing import IntoDataFrame from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, @@ -31,6 +33,20 @@ "pow", ] +_NUMPY_OPS = { + "add": np.add, + "sub": np.subtract, + "mul": np.multiply, + "div": np.divide, + "truediv": np.true_divide, + "floordiv": np.floor_divide, + "mod": np.mod, + "pow": np.power, +} + +# these can divide by zero; fill_value handling applies only to them. +_DIVISION_LIKE = {"div", "truediv", "floordiv", "mod"} + @Substitution( variables=_variables_numerical_docstring, @@ -54,12 +70,10 @@ class RelativeFeatures(BaseCreation): features to / by a group of reference variables. The features resulting from these functions are added to the dataframe. - This transformer works only with numerical variables. It uses the pandas methods - `pd.DataFrame.add`, `pd.DataFrame.sub`, `pd.DataFrame.mul`, `pd.DataFrame.div`, - `pd.DataFrame.truediv`, `pd.DataFrame.floordiv`, `pd.DataFrame.mod` and - `pd.DataFrame.pow`. - Find out more in `pandas documentation - `_. + This transformer works only with numerical variables. It uses NumPy's `add`, + `subtract`, `multiply`, `divide`, `true_divide`, `floor_divide`, `mod` and + `power` under the hood, matching the semantics of the equivalent pandas + `DataFrame.add`, `DataFrame.sub`, etc. methods. More details in the :ref:`User Guide `. @@ -125,6 +139,27 @@ class RelativeFeatures(BaseCreation): 0 1 4 3 0.333333 1.333333 1 2 5 4 0.500000 1.250000 2 3 6 5 0.600000 1.200000 + + With polars: + + >>> import polars as pl + >>> from feature_engine.creation import RelativeFeatures + >>> X = pl.DataFrame({"x1": [1, 2, 3], "x2": [4, 5, 6], "x3": [3, 4, 5]}) + >>> rf = RelativeFeatures(variables=["x1", "x2"], + >>> reference=["x3"], + >>> func=["div"]) + >>> rf.fit(X) + >>> rf.transform(X) + shape: (3, 5) + ┌─────┬─────┬─────┬───────────┬───────────┐ + │ x1 ┆ x2 ┆ x3 ┆ x1_div_x3 ┆ x2_div_x3 │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ i64 ┆ i64 ┆ f64 ┆ f64 │ + ╞═════╪═════╪═════╪═══════════╪═══════════╡ + │ 1 ┆ 4 ┆ 3 ┆ 0.333333 ┆ 1.333333 │ + │ 2 ┆ 5 ┆ 4 ┆ 0.5 ┆ 1.25 │ + │ 3 ┆ 6 ┆ 5 ┆ 0.6 ┆ 1.2 │ + └─────┴─────┴─────┴───────────┴───────────┘ """ def __init__( @@ -179,124 +214,70 @@ def __init__( self.func = func self.fill_value = fill_value - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Add new features. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe + X_new: dataframe The input dataframe plus the new variables. """ X = self._check_transform_input_and_state(X) - methods_dict = { - "add": self._add, - "mul": self._mul, - "sub": self._sub, - "div": self._div, - "truediv": self._truediv, - "floordiv": self._floordiv, - "mod": self._mod, - "pow": self._pow, - } + nw_X = nw.from_native(X, eager_only=True) + # Extract each column as its own 1D array (not one batched 2D array + # via select().to_numpy()) so mixed int/float variables each keep + # their own dtype promotion, matching pandas' per-column .sub()/ + # .div()/etc. instead of upcasting everything to a common dtype. + var_arrays = {var: nw_X.get_column(var).to_numpy() for var in self.variables} + ref_arrays = {ref: nw_X.get_column(ref).to_numpy() for ref in self.reference} + new_series = [] for func in self.func: - methods_dict[func](X) - - if self.drop_original: - X.drop( - columns=set(self.variables + self.reference), - inplace=True, - ) - - return X - - def _sub(self, X): - for reference in self.reference: - varname = [f"{var}_sub_{reference}" for var in self.variables] - X[varname] = X[self.variables].sub(X[reference], axis=0) - return X - - def _add(self, X): - for reference in self.reference: - varname = [f"{var}_add_{reference}" for var in self.variables] - X[varname] = X[self.variables].add(X[reference], axis=0) - return X - - def _mul(self, X): - for reference in self.reference: - varname = [f"{var}_mul_{reference}" for var in self.variables] - X[varname] = X[self.variables].mul(X[reference], axis=0) - return X - - def _div(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_div_{reference}" for var in self.variables] - X[varname] = X[self.variables].div(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _truediv(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_truediv_{reference}" for var in self.variables] - X[varname] = X[self.variables].truediv(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _floordiv(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_floordiv_{reference}" for var in self.variables] - X[varname] = X[self.variables].floordiv(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _mod(self, X): - for reference in self.reference: - zeros_ix, contains_zero = self._find_zeroes_in_reference(X, reference) - - if self.fill_value is None and contains_zero: - self._raise_error_when_zero_in_denominator() - - varname = [f"{var}_mod_{reference}" for var in self.variables] - X[varname] = X[self.variables].mod(X[reference], axis=0) - - if contains_zero: - X.loc[zeros_ix, varname] = self.fill_value - return X - - def _pow(self, X): - for reference in self.reference: - varname = [f"{var}_pow_{reference}" for var in self.variables] - X[varname] = X[self.variables].pow(X[reference], axis=0) - return X + op = _NUMPY_OPS[func] + for reference in self.reference: + ref_arr = ref_arrays[reference] + + if func in _DIVISION_LIKE: + zero_mask = ref_arr == 0 + contains_zero = zero_mask.any() + if self.fill_value is None and contains_zero: + self._raise_error_when_zero_in_denominator() + + for var in self.variables: + name = f"{var}_{func}_{reference}" + if func in _DIVISION_LIKE: + with np.errstate(divide="ignore", invalid="ignore"): + result = op(var_arrays[var], ref_arr) + if contains_zero: + # floordiv/mod on integer input stay integer-typed; + # widen to match fill_value if it wouldn't fit, + # mirroring pandas' automatic dtype promotion. + fill_arr = np.asarray(self.fill_value) + if not np.can_cast(fill_arr, result.dtype, casting="safe"): + result = result.astype( + np.result_type(result.dtype, fill_arr.dtype) + ) + result[zero_mask] = self.fill_value + else: + result = op(var_arrays[var], ref_arr) + + new_series.append( + nw.new_series(name, result, backend=nw_X.implementation) + ) + + nw_X = nw_X.with_columns(*new_series) + if self.drop_original is True: + nw_X = nw_X.drop(list(set(self.variables + self.reference))) + + return nw_X.to_native() def _raise_error_when_zero_in_denominator(self): raise ValueError( @@ -305,11 +286,6 @@ def _raise_error_when_zero_in_denominator(self): "or set `fill_value` to a number." ) - def _find_zeroes_in_reference(self, X, var): - zero_ix = X[var] == 0 - zero_bool = (zero_ix).any() - return zero_ix, zero_bool - def _get_new_features_name(self) -> List: """Return names of the created features.""" diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index fe313a627..b3a07be12 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -4,118 +4,84 @@ from typing import List, Tuple, Union +import narwhals as nw +import narwhals.dependencies as nwd +import narwhals.selectors as nws import numpy as np -import pandas as pd -from scipy.sparse import issparse +from narwhals.typing import IntoDataFrame, IntoDataFrameT, IntoSeries from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d -from feature_engine.variable_handling._variable_type_checks import is_object - -def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: +def check_X(X: IntoDataFrameT) -> IntoDataFrameT: """ - Checks if the input is a DataFrame and then creates a copy. This is an important - step not to accidentally transform the original dataset entered by the user. - - If the input is a numpy array, it converts it to a pandas Dataframe. The column - names are strings representing the column index starting at 0. - - Feature-engine was originally designed to work with pandas dataframes. However, - allowing numpy arrays as input allows 2 things: - - We can use the Scikit-learn tests for transformers provided by the - `check_estimator` function to test the compatibility of our transformers with - sklearn functionality. - - Feature-engine transformers can be used within a Scikit-learn Pipeline together - with Scikit-learn transformers like the `SimpleImputer`, which return by default - Numpy arrays. + Checks that X is a dataframe from any library supported by narwhals (for example + pandas, polars, modin, cuDF, or PyArrow). Parameters ---------- - X : pandas Dataframe or numpy array. - The input to check and copy or transform. + X : dataframe (pandas, polars, PyArrow, modin, or cuDF). Feature-engine does + not support libraries that build a deferred query plan (for example Dask, + DuckDB, PySpark, Ibis, or a polars LazyFrame). Convert those to an eager + dataframe (e.g. `LazyFrame.collect()`) before passing them in. + The input to check and transform. Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array. + If the input is not a recognised dataframe. ValueError - If the input is an empty dataframe. + If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : pandas Dataframe. - A copy of original DataFrame or a converted Numpy array. + X : dataframe. + The validated dataframe in its native format. """ - if isinstance(X, pd.DataFrame): - if not X.columns.is_unique: - raise ValueError("Input data contains duplicated variable names.") - X = X.copy() - - elif isinstance(X, (np.generic, np.ndarray)): - # If input is scalar raise error - if X.ndim == 0: - raise ValueError( - "Expected 2D array, got scalar array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) - ) - # If input is 1D raise error - if X.ndim == 1: + if nwd.is_into_dataframe(X): + # from_native() raises narwhals.exceptions.DuplicateError, a ValueError + # subclass, when the dataframe has duplicated column names. + nw_X = nw.from_native(X, eager_only=True) + if nw_X.is_empty() or nw_X.shape[1] == 0: raise ValueError( - "Expected 2D array, got 1D array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) + f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " + "minimum of 1 is required." ) - if np.any(np.iscomplex(X)): - raise TypeError("Complex data not supported by this transformer.") - - X = pd.DataFrame(X) - X.columns = [f"x{i}" for i in range(X.shape[1])] - - elif issparse(X): - raise TypeError("This transformer does not support sparse matrices.") - else: raise TypeError( - f"X must be a numpy array or pandas dataframe. Got {type(X)} instead." + "X must be a dataframe from a library supported by narwhals " + f"(e.g. pandas, polars, PyArrow). Got {type(X)} instead." ) - if X.empty: - raise ValueError( - "0 feature(s) (shape=%s) while a minimum of %d is required." % (X.shape, 1) - ) - - return X + return nw_X.to_native() def check_y( - y: Union[np.generic, np.ndarray, pd.Series, pd.DataFrame, List], + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> pd.Series: +): """ - Checks that y is a series or a dataframe, or alternatively, if it can be converted - to a series or dataframe. + Checks that y is a Series or DataFrame from a library supported by narwhals (for + example pandas or polars), or alternatively, if it can be converted to a numpy + array. Parameters ---------- - y : pd.Series, pd.DataFrame, np.array, list - The input to check and copy or transform. + y : Series or DataFrame (pandas, polars, PyArrow, modin, or cuDF), np.array, + list. Feature-engine does not support libraries that build a deferred + query plan (for example Dask, DuckDB, PySpark, Ibis, or a polars + LazyFrame). Convert those to an eager dataframe (e.g. `LazyFrame.collect()`) + before passing them in. + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Returns ------- - y: pd.Series or pd.DataFrame + y: Series, DataFrame, or numpy array """ - if y is None: raise ValueError( "requires y to be passed, but the target y is None", @@ -123,110 +89,97 @@ def check_y( "y should be a 1d array", ) - elif isinstance(y, pd.Series): - if y.isnull().any(): + if nwd.is_into_series(y): + nw_y = nw.from_native(y, series_only=True) + if nw_y.is_null().any() or ( + nw_y.dtype.is_numeric() and nw_y.is_nan().any() + ): raise ValueError("y contains NaN values.") - if not is_object(y) and not np.isfinite(y).all(): - raise ValueError("y contains infinity values.") - if y_numeric and is_object(y): - y = y.astype("float64") - y = y.copy() - - elif isinstance(y, pd.DataFrame): - if y.isnull().any().any(): + if nw_y.dtype.is_numeric(): + if not np.isfinite(nw_y.to_numpy()).all(): + raise ValueError("y contains infinity values.") + elif y_numeric: + nw_y = nw_y.cast(nw.Float64()) + return nw_y.to_native() + + if nwd.is_into_dataframe(y): + nw_y = nw.from_native(y, eager_only=True) + if ( + nw_y.select(nw.all().is_null().any()).to_numpy().any() + or nw_y.select(nws.numeric().is_nan().any()).to_numpy().any() + ): raise ValueError("y contains NaN values.") - if not np.isfinite(y).all().all(): + if not np.isfinite(nw_y.to_numpy()).all(): raise ValueError("y contains infinity values.") - y = y.copy() + return nw_y.to_native() - else: - try: - y = column_or_1d(y) - y = _check_y(y, multi_output=False, y_numeric=y_numeric) - y = pd.Series(y).copy() - except ValueError: - y = _check_y(y, multi_output=True, y_numeric=y_numeric) - y = pd.DataFrame(y).copy() - return y + try: + y = column_or_1d(y) + return _check_y(y, multi_output=False, y_numeric=y_numeric) + except ValueError: + return _check_y(y, multi_output=True, y_numeric=y_numeric) def check_X_y( - X: Union[np.generic, np.ndarray, pd.DataFrame], - y: Union[np.generic, np.ndarray, pd.Series, List], + X: IntoDataFrameT, + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> Tuple[pd.DataFrame, pd.Series]: +) -> Tuple[IntoDataFrameT, Union[IntoSeries, IntoDataFrame, np.ndarray]]: """ - Ensures X and y are compatible pandas DataFrame and Series. If both are pandas - objects, checks that their indexes match. If any is a numpy array, converts to - pandas object with compatible index. - - This transformer ensures that we can concatenate X and y using `pandas.concat`, - functionality needed in the encoders. + Ensures X and y are compatible dataframe/array-like objects with a consistent + number of rows. If both are pandas objects, checks that their indexes match. Parameters ---------- - X: Pandas DataFrame or numpy ndarray - The input to check and copy or transform. + X: dataframe (pandas, polars, PyArrow, modin, or cuDF). Feature-engine does + not support libraries that build a deferred query plan (for example Dask, + DuckDB, PySpark, Ibis, or a polars LazyFrame). Convert those to an eager + dataframe (e.g. `LazyFrame.collect()`) before passing them in. + The input to check. - y: pd.Series, np.array, list - The input to check and copy or transform. + y: Series, DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Raises ------ - ValueError: if X and y are pandas objects with inconsistent indexes. - TypeError: if X is sparse matrix, empty dataframe or not a dataframe. - TypeError: if y can't be parsed as pandas Series. + TypeError + If X is not a recognised dataframe. + ValueError + If X has duplicated column names, 0 columns, or 0 rows; if y is None, or + contains NaN or infinity values; if X and y have a different number of + rows; or if X and y are pandas objects with mismatched indexes. Returns ------- - X: Pandas DataFrame - y: Pandas Series + X: dataframe + y: Series, DataFrame, or numpy array """ + X = check_X(X) + y = check_y(y, y_numeric=y_numeric) + check_consistent_length(X, y) - def _check_X_y(X, y): - X = check_X(X) - y = check_y(y, y_numeric=y_numeric) - check_consistent_length(X, y) - return X, y - - # case 1: both are pandas objects - if isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - # Check that their indexes match. - if X.index.equals(y.index) is False: - raise ValueError("The indexes of X and y do not match.") - - # case 2: X is dataframe and y is something else - if isinstance(X, pd.DataFrame) and not isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - y.index = X.index - - # case 3: X is not a dataframe and y is a series - elif not isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - X.index = y.index - - # all other cases - else: - X, y = _check_X_y(X, y) + if nwd.is_pandas_dataframe(X): + if nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y): + if not X.index.equals(y.index): + raise ValueError("The indexes of X and y do not match.") return X, y -def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: +def _check_X_matches_training_df(X: IntoDataFrame, reference: int) -> None: """ - Checks that DataFrame to transform has the same number of columns that the - DataFrame used with the fit() method. + Checks that the dataframe to transform has the same number of columns as the + dataframe used with the fit() method. Parameters ---------- - X : Pandas DataFrame - The df to be checked + X : dataframe (pandas, polars, or any other library supported by narwhals) + The df to be checked. reference : int The number of columns in the dataframe that was used with the fit() method. @@ -234,92 +187,78 @@ def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: ------ ValueError If the number of columns does not match. - - Returns - ------- - None """ - if X.shape[1] != reference: raise ValueError( "The number of columns in this dataset is different from the one used to " "fit this transformer (when using the fit() method)." ) - return None - def _check_contains_na( - X: pd.DataFrame, + X: IntoDataFrame, variables: List[Union[str, int]], + error_msg: str = "simple", ) -> None: """ - Checks if DataFrame contains null values in the selected columns. + Checks if the dataframe contains null values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List The selected group of variables in which null values will be examined. - Raises - ------ - ValueError - If the variable(s) contain null values. - """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer." - ) - - -def _check_optional_contains_na( - X: pd.DataFrame, variables: List[Union[str, int]] -) -> None: - """ - Checks if DataFrame contains null values in the selected columns. - - Parameters - ---------- - X : Pandas DataFrame - - variables : List - The selected group of variables in which null values will be examined. + error_msg : str, default="simple" + The message in the error. Some transformers can ignore null values. Raises ------ ValueError If the variable(s) contain null values. """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - - -def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> None: + error_msg_simple = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + error_msg_ignore = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) + nw_X = nw.from_native(X, eager_only=True) + if nwd.is_pandas_dataframe(X): + numeric_vars = list(X[variables].select_dtypes(include="number").columns) + else: + numeric_vars = nw_X.select(variables).select(nw.selectors.numeric()).columns + if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any() or ( + numeric_vars + and nw_X.select(nw.col(numeric_vars).is_nan().any()).to_numpy().any() + ): + if error_msg == "simple": + raise ValueError(error_msg_simple) + else: + raise ValueError(error_msg_ignore) + + +def _check_contains_inf(X: IntoDataFrame, variables: List[Union[str, int]]) -> None: """ - Checks if DataFrame contains inf values in the selected columns. + Checks if the dataframe contains inf values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List - The selected group of variables in which null values will be examined. + The selected group of variables in which infinite values will be examined. Raises ------ ValueError If the variable(s) contain np.inf values """ - - if np.isinf(X[variables]).any().any(): + values = nw.from_native(X, eager_only=True).select(nw.col(variables)).to_numpy() + if np.isinf(values.astype(float)).any(): raise ValueError( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 35427f260..53eca3095 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -20,7 +20,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -123,7 +123,7 @@ class CategoricalMethodsMixin(TransformerMixin, BaseEstimator, GetFeatureNamesOu def _check_na(self, X: pd.DataFrame, variables): if self.missing_values == "raise": - _check_optional_contains_na(X, variables) + _check_contains_na(X, variables, error_msg="optional") def _check_or_select_variables(self, X: pd.DataFrame): """ @@ -225,7 +225,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") X = self._encode(X) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 84c1f6910..2bbd2bf73 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -23,7 +23,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -255,7 +255,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") with_nan = [] else: with_nan = [np.nan] diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index c438487d6..f15f87003 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -17,7 +17,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixin, CategoricalMethodsMixin, @@ -247,7 +247,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # if data contains nan, fail before running any logic if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.encoder_dict_ = {} @@ -317,7 +317,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: check_is_fitted(self) X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") if len(self.variables_) == 0: return X diff --git a/feature_engine/preprocessing/match_categories.py b/feature_engine/preprocessing/match_categories.py index e0e863c1f..9d66c3d6c 100644 --- a/feature_engine/preprocessing/match_categories.py +++ b/feature_engine/preprocessing/match_categories.py @@ -19,7 +19,7 @@ ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -148,7 +148,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): variables_ = self._check_or_select_variables(X) if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.category_dict_ = dict() for var in variables_: @@ -175,7 +175,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") for feature, levels in self.category_dict_.items(): X[feature] = pd.Categorical( diff --git a/feature_engine/tags.py b/feature_engine/tags.py index ad36b030a..0c15bb1a0 100644 --- a/feature_engine/tags.py +++ b/feature_engine/tags.py @@ -1,9 +1,3 @@ -import sklearn -from sklearn.utils.fixes import parse_version - -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - - def _return_tags(): tags = { "preserves_dtype": [], @@ -32,14 +26,13 @@ def _return_tags(): }, } - if sklearn_version > parse_version("1.6"): - msg1 = "against Feature-engines design." - msg2 = "Our transformers do not preserve dtype." - all_fail = { - "check_do_not_raise_errors_in_init_or_set_params": msg1, - "check_transformer_preserve_dtypes": msg2, - # TODO: investigate this test further. - "check_n_features_in_after_fitting": "not sure why it fails, we do check.", - } - tags["_xfail_checks"].update(all_fail) # type: ignore + msg1 = "against Feature-engines design." + msg2 = "Our transformers do not preserve dtype." + all_fail = { + "check_do_not_raise_errors_in_init_or_set_params": msg1, + "check_transformer_preserve_dtypes": msg2, + # TODO: investigate this test further. + "check_n_features_in_after_fitting": "not sure why it fails, we do check.", + } + tags["_xfail_checks"].update(all_fail) # type: ignore return tags diff --git a/feature_engine/text/text_features.py b/feature_engine/text/text_features.py index 96dd283c0..5198639d3 100644 --- a/feature_engine/text/text_features.py +++ b/feature_engine/text/text_features.py @@ -12,7 +12,7 @@ _check_param_missing_values, ) from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -236,7 +236,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) # Set features to extract if self.features is None: @@ -278,7 +280,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) else: X[self.variables_] = X[self.variables_].fillna("") diff --git a/feature_engine/transformation/arcsin.py b/feature_engine/transformation/arcsin.py index da4045fa4..67d0dad59 100644 --- a/feature_engine/transformation/arcsin.py +++ b/feature_engine/transformation/arcsin.py @@ -3,8 +3,9 @@ from typing import List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -105,6 +106,30 @@ class ArcsinTransformer(BaseNumericalTransformer): 2 0.144664 3 0.783236 4 0.650777 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import ArcsinTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.beta(1, 1, size=6))}) + >>> ast = ArcsinTransformer() + >>> ast.fit(X) + >>> ast.transform(X) + shape: (6, 1) + ┌──────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.785437 │ + │ 0.253389 │ + │ 0.144664 │ + │ 0.783236 │ + │ 0.650777 │ + │ 0.883313 │ + └──────────┘ """ def __init__( @@ -118,17 +143,17 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn parameters. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ @@ -136,7 +161,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): X, variables_ = self._fit_setup(X) # check if the variables are in the correct range - if ((X[variables_] < 0) | (X[variables_] > 1)).any().any(): + values = nw.from_native(X, eager_only=True).select(variables_).to_numpy() + if np.any((values < 0) | (values > 1)): raise ValueError( "Some variables contain values outside the possible range 0-1. " "Can't apply the arcsin transformation. " @@ -147,52 +173,68 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Apply the arcsin transformation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy() + # check if the variables are in the correct range - if ((X[self.variables_] < 0) | (X[self.variables_] > 1)).any().any(): + if np.any((values < 0) | (values > 1)): raise ValueError( "Some variables contain values outside the possible range 0-1. " "Can't apply the arcsin transformation." ) # transform - X.loc[:, self.variables_] = np.arcsin(np.sqrt(X.loc[:, self.variables_])) + result = np.arcsin(np.sqrt(values)) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the transformed variables. """ + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy() + # inverse_transform - X.loc[:, self.variables_] = (np.sin(X.loc[:, self.variables_])) ** 2 + result = np.sin(values) ** 2 + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/feature_engine/transformation/arcsinh.py b/feature_engine/transformation/arcsinh.py index 92ebf2c0a..0a31fc7ae 100644 --- a/feature_engine/transformation/arcsinh.py +++ b/feature_engine/transformation/arcsinh.py @@ -3,8 +3,9 @@ from typing import List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -119,11 +120,35 @@ class ArcSinhTransformer(BaseNumericalTransformer): >>> X = ast.transform(X) >>> X.head() x - 0 7.516076 - 1 -6.330816 - 2 7.780254 - 3 8.825252 - 4 -6.995893 + 0 6.901163 + 1 -5.622327 + 2 7.166558 + 3 8.021604 + 4 -6.149128 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import ArcSinhTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.randn(6) * 1000)}) + >>> ast = ArcSinhTransformer() + >>> ast.fit(X) + >>> ast.transform(X) + shape: (6, 1) + ┌───────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞═══════════╡ + │ 6.901163 │ + │ -5.622327 │ + │ 7.166558 │ + │ 8.021604 │ + │ -6.149128 │ + │ -6.149058 │ + └───────────┘ """ def __init__( @@ -152,17 +177,17 @@ def __init__( self.loc = float(loc) self.scale = float(scale) - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Selects the numerical variables and stores feature names. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. Returns @@ -179,46 +204,48 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Transform the variables using the arcsinh function. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) - # Ensure float dtype for the transformation - X[self.variables_] = X[self.variables_].astype(float) - # Apply arcsinh transformation: arcsinh((x - loc) / scale) - X.loc[:, self.variables_] = np.arcsinh( - (X.loc[:, self.variables_] - self.loc) / self.scale - ) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + result = np.arcsinh((values - self.loc) / self.scale) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be inverse transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the inverse transformed variables. """ @@ -226,9 +253,14 @@ def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._check_transform_input_and_state(X) # Inverse transform: x = sinh(y) * scale + loc - X.loc[:, self.variables_] = ( - np.sinh(X.loc[:, self.variables_]) * self.scale + self.loc - ) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + result = np.sinh(values) * self.scale + self.loc + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/feature_engine/transformation/boxcox.py b/feature_engine/transformation/boxcox.py index 52d5feffb..fa1b64391 100644 --- a/feature_engine/transformation/boxcox.py +++ b/feature_engine/transformation/boxcox.py @@ -3,9 +3,11 @@ from typing import List, Optional, Union -import pandas as pd +import narwhals as nw +import numpy as np +import scipy.special as spsp import scipy.stats as stats -from scipy.special import inv_boxcox +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -119,6 +121,30 @@ class BoxCoxTransformer(BaseNumericalTransformer): 2 0.662654 3 1.607518 4 -0.232237 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import BoxCoxTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.lognormal(size=6))}) + >>> bct = BoxCoxTransformer() + >>> bct.fit(X) + >>> bct.transform(X) + shape: (6, 1) + ┌───────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞═══════════╡ + │ 0.403681 │ + │ -0.146883 │ + │ 0.495725 │ + │ 0.845914 │ + │ -0.259585 │ + │ -0.259565 │ + └───────────┘ """ def __init__( @@ -132,27 +158,31 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the optimal lambda for the BoxCox transformation. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ # check input dataframe X, variables_ = self._fit_setup(X) - lambda_dict_ = {} + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(variables_).to_numpy().astype(float) - for var in variables_: - _, lambda_dict_[var] = stats.boxcox(X[var]) + lambda_dict_ = {} + # lambda search is per-column and not vectorizable across columns, + # unlike transform()'s elementwise application once lambdas are known + for i, var in enumerate(variables_): + _, lambda_dict_[var] = stats.boxcox(values[:, i]) self.variables_ = variables_ self.lambda_dict_ = lambda_dict_ @@ -160,55 +190,71 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Apply the BoxCox transformation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + # check contains zero or negative values - if (X[self.variables_] <= 0).any().any(): + if (values <= 0).any(): raise ValueError("Data must be positive.") # transform - for feature in self.variables_: - X[feature] = stats.boxcox(X[feature], lmbda=self.lambda_dict_[feature]) + lmbdas = np.array([self.lambda_dict_[var] for var in self.variables_]) + result = spsp.boxcox(values, lmbdas) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be inverse transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the original variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + # inverse transform - for feature in self.variables_: - X[feature] = inv_boxcox(X[feature], self.lambda_dict_[feature]) + lmbdas = np.array([self.lambda_dict_[var] for var in self.variables_]) + result = spsp.inv_boxcox(values, lmbdas) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/feature_engine/transformation/log.py b/feature_engine/transformation/log.py index 9cd12c307..0ee13ee35 100644 --- a/feature_engine/transformation/log.py +++ b/feature_engine/transformation/log.py @@ -4,8 +4,9 @@ import warnings from typing import Dict, List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._base_transformers.mixins import FitFromDictMixin @@ -126,6 +127,30 @@ class LogTransformer(BaseNumericalTransformer, FitFromDictMixin): 2 0.647689 3 1.523030 4 -0.234153 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import LogTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.lognormal(size=6))}) + >>> lt = LogTransformer() + >>> lt.fit(X) + >>> lt.transform(X) + shape: (6, 1) + ┌───────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞═══════════╡ + │ 0.496714 │ + │ -0.138264 │ + │ 0.647689 │ + │ 1.52303 │ + │ -0.234153 │ + │ -0.234137 │ + └───────────┘ """ def __init__( @@ -154,7 +179,7 @@ def __init__( self.base = base self.C = C - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the constant C to add to the variable before the logarithm transformation, if C="auto". Otherwise, this transformer does not learn @@ -162,11 +187,11 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ @@ -176,21 +201,21 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): else: X, variables_ = self._fit_setup(X) + values = nw.from_native(X, eager_only=True).select(variables_).to_numpy() + values = values.astype(float) + C_ = self.C - # calculate C to add to each variable + # 0 for strictly positive variables, abs(min) + 1 (shift to positive) + # otherwise. if self.C == "auto": - # we add 0 to positive variables - c_dict = {var: 0 for var in variables_ if X[var].min() > 0} - - # we add the minimum plus 1 to non-positive variables - non_positive_vars = [var for var in variables_ if var not in c_dict.keys()] - c_dict.update(dict(X[non_positive_vars].min(axis=0).abs() + 1)) - C_ = c_dict # type:ignore + mins = values.min(axis=0) + c_values = np.where(mins > 0, 0, np.abs(mins) + 1) + C_ = dict(zip(variables_, c_values.tolist())) # C=0 is the original LogTransformer contract: no constant is added, # so fail fast at fit time exactly as before this class supported C. - if C_ == 0 and (X[variables_] <= 0).any().any(): + if C_ == 0 and np.any(values <= 0): raise ValueError( "Some variables contain zero or negative values, can't apply log" ) @@ -201,18 +226,25 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def _c_as_array(self) -> Union[int, float, np.ndarray]: + """Broadcastable form of C_: a plain scalar, or a numpy array ordered + to line up column-wise with self.variables_ when C_ is a dict.""" + if isinstance(self.C_, dict): + return np.array([self.C_[var] for var in self.variables_], dtype=float) + return self.C_ + + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Transform the variables with the logarithm of x plus the constant C. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ @@ -229,42 +261,60 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: + " constant C, can't apply log." ) - if (X[self.variables_] + self.C_ <= 0).any().any(): - raise ValueError(error_msg) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + shifted = values + self._c_as_array() - X[self.variables_] = X[self.variables_].astype(float) + if np.any(shifted <= 0): + raise ValueError(error_msg) # transform if self.base == "e": - X.loc[:, self.variables_] = np.log(X.loc[:, self.variables_] + self.C_) - elif self.base == "10": - X.loc[:, self.variables_] = np.log10(X.loc[:, self.variables_] + self.C_) + result = np.log(shifted) + else: + result = np.log10(shifted) + + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + c_arr = self._c_as_array() + # inverse_transform if self.base == "e": - X.loc[:, self.variables_] = np.exp(X.loc[:, self.variables_]) - self.C_ - elif self.base == "10": - X.loc[:, self.variables_] = 10 ** X.loc[:, self.variables_] - self.C_ + result = np.exp(values) - c_arr + else: + result = 10**values - c_arr + + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/feature_engine/transformation/power.py b/feature_engine/transformation/power.py index 89aea9bf2..5cb376d66 100644 --- a/feature_engine/transformation/power.py +++ b/feature_engine/transformation/power.py @@ -3,8 +3,9 @@ from typing import List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -99,6 +100,30 @@ class PowerTransformer(BaseNumericalTransformer): 2 1.382432 3 2.141518 4 0.889517 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import PowerTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.lognormal(size=6))}) + >>> pt = PowerTransformer() + >>> pt.fit(X) + >>> pt.transform(X) + shape: (6, 1) + ┌──────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 1.281918 │ + │ 0.933203 │ + │ 1.382432 │ + │ 2.141518 │ + │ 0.889517 │ + │ 0.889524 │ + └──────────┘ """ def __init__( @@ -117,17 +142,17 @@ def __init__( self.return_empty = return_empty self.exp = exp - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn parameters. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] - The training input samples. - Can be the entire dataframe, not just the variables to transform. + X: dataframe of shape = [n_samples, n_features]. + The training input samples. Can be the entire dataframe, not just the + variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ @@ -139,49 +164,64 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Apply the power transformation to the variables. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas Dataframe + X_new: dataframe The dataframe with the power transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + # transform - X[self.variables_] = X[self.variables_].astype(float) - X.loc[:, self.variables_] = np.power(X.loc[:, self.variables_], self.exp) + result = np.power(values, self.exp) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas Dataframe + X_tr: dataframe The dataframe with the power transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + # inverse_transform - X.loc[:, self.variables_] = np.power(X.loc[:, self.variables_], 1 / self.exp) + result = np.power(values, 1 / self.exp) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/feature_engine/transformation/reciprocal.py b/feature_engine/transformation/reciprocal.py index 22678544c..1541cdaaf 100644 --- a/feature_engine/transformation/reciprocal.py +++ b/feature_engine/transformation/reciprocal.py @@ -3,7 +3,9 @@ from typing import List, Optional, Union -import pandas as pd +import narwhals as nw +import numpy as np +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -97,6 +99,30 @@ class ReciprocalTransformer(BaseNumericalTransformer): 2 0.115164 3 0.110047 4 0.101726 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import ReciprocalTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(10 - np.random.exponential(size=6))}) + >>> rt = ReciprocalTransformer() + >>> rt.fit(X) + >>> rt.transform(X) + shape: (6, 1) + ┌──────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.104924 │ + │ 0.143064 │ + │ 0.115164 │ + │ 0.110047 │ + │ 0.101726 │ + │ 0.101725 │ + └──────────┘ """ def __init__( @@ -109,17 +135,17 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn parameters. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ @@ -127,7 +153,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): X, variables_ = self._fit_setup(X) # check if the variables contain the value 0 - if (X[variables_] == 0).any().any(): + values = nw.from_native(X, eager_only=True).select(variables_).to_numpy() + if np.any(values == 0): raise ValueError( "Some variables contain the value zero, can't apply reciprocal " "transformation." @@ -138,49 +165,56 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Apply the reciprocal 1 / x transformation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy() + # check if the variables contain the value 0 - if (X[self.variables_] == 0).any().any(): + if np.any(values == 0): raise ValueError( "Some variables contain the value zero, can't apply reciprocal " "transformation." ) # transform - X[self.variables_] = X[self.variables_].astype(float) - X.loc[:, self.variables_] = 1 / X.loc[:, self.variables_] + result = 1 / values + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the transformed variables. """ # inverse_transform diff --git a/feature_engine/transformation/yeojohnson.py b/feature_engine/transformation/yeojohnson.py index 82fa53dac..54da3001b 100644 --- a/feature_engine/transformation/yeojohnson.py +++ b/feature_engine/transformation/yeojohnson.py @@ -3,9 +3,10 @@ from typing import List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd import scipy.stats as stats +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -108,11 +109,35 @@ class YeoJohnsonTransformer(BaseNumericalTransformer): >>> X = yjt.transform(X) >>> X.head() x - 0 -267042.906453 - 1 -444357.138990 - 2 -221626.115742 - 3 -23647.632651 - 4 -467264.993249 + 0 -267042.661354 + 1 -444356.715596 + 2 -221625.915167 + 3 -23647.614887 + 4 -467264.546413 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import YeoJohnsonTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.lognormal(size=6) - 10)}) + >>> yjt = YeoJohnsonTransformer() + >>> yjt.fit(X) + >>> yjt.transform(X) + shape: (6, 1) + ┌────────────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞════════════════╡ + │ -467714.164249 │ + │ -795057.401919 │ + │ -385148.281012 │ + │ -37417.353351 │ + │ -837807.71099 │ + │ -837800.580457 │ + └────────────────┘ """ def __init__( @@ -125,27 +150,31 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the optimal lambda for the Yeo-Johnson transformation. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ # check input dataframe X, variables_ = self._fit_setup(X) - lambda_dict_ = {} + values = nw.from_native(X, eager_only=True).select(variables_).to_numpy() + values = values.astype(float) - for var in variables_: - _, lambda_dict_[var] = stats.yeojohnson(X[var]) + # scipy searches the optimal lambda one column at a time, there is no + # vectorized multi-column form of the search. + lambda_dict_ = {} + for i, var in enumerate(variables_): + _, lambda_dict_[var] = stats.yeojohnson(values[:, i]) self.variables_ = variables_ self.lambda_dict_ = lambda_dict_ @@ -153,55 +182,77 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Apply the Yeo-Johnson transformation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted - X = self._check_transform_input_and_state(X) - for feature in self.variables_: - X[feature] = stats.yeojohnson(X[feature], lmbda=self.lambda_dict_[feature]) + + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + + # transform + result = np.empty_like(values) + for i, var in enumerate(self.variables_): + result[:, i] = stats.yeojohnson(values[:, i], lmbda=self.lambda_dict_[var]) + + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) - for feature in self.variables_: - X[feature] = self._inverse_transform_series( - X[feature], lmbda=self.lambda_dict_[feature] + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + + # inverse_transform + result = np.empty_like(values) + for i, var in enumerate(self.variables_): + result[:, i] = self._inverse_transform_array( + values[:, i], lmbda=self.lambda_dict_[var] ) + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() + return X - def _inverse_transform_series(self, X: pd.Series, lmbda: float) -> pd.Series: - x_inv = pd.Series(np.zeros_like(X), index=X.index) + def _inverse_transform_array(self, X: np.ndarray, lmbda: float) -> np.ndarray: + x_inv = np.zeros_like(X) pos = X >= 0 # when x >= 0 diff --git a/feature_engine/variable_handling/_variable_type_checks.py b/feature_engine/variable_handling/_variable_type_checks.py index 17eb4e41d..a787178df 100644 --- a/feature_engine/variable_handling/_variable_type_checks.py +++ b/feature_engine/variable_handling/_variable_type_checks.py @@ -1,62 +1,113 @@ -import pandas as pd -from pandas.api.types import is_object_dtype, is_string_dtype -from pandas.core.dtypes.common import is_datetime64_any_dtype as is_datetime -from pandas.core.dtypes.common import is_numeric_dtype as is_numeric +import warnings +from datetime import date, datetime +import narwhals as nw +from dateutil.parser import parser -def is_object(s) -> bool: - return is_object_dtype(s) or is_string_dtype(s) +def _is_date_or_datetime(dtype) -> bool: + # nw.selectors.datetime() only matches Datetime, not Date, so this needs + # its own explicit check. + return isinstance(dtype, (nw.Date, nw.Datetime)) -def _is_categorical_and_is_not_datetime(column: pd.Series) -> bool: - # check for datetime only if the type of the categories is not numeric - # because pd.to_datetime throws an error when it is an integer - if isinstance(column.dtype, pd.CategoricalDtype): - is_cat = _is_categories_num(column) or not _is_convertible_to_dt(column) - # check for datetime only if object cannot be cast as numeric because - # if it could pd.to_datetime would convert it to datetime regardless - elif is_object(column): - is_cat = _is_convertible_to_num(column) or not _is_convertible_to_dt(column) - - else: - is_cat = False - - return is_cat +def _looks_like_date_string(value) -> bool: + # taken from pandas + # https://github.com/pandas-dev/pandas/blob/cbae8aea4a31a4052736ab0d23f284ff1e78aa06/pandas/_libs/tslibs/parsing.pyx#L666 + try: + result, _ = parser()._parse(value) + except TypeError: + return False + if result is None: + return False -def _is_categories_num(column: pd.Series) -> bool: - return is_numeric(column.dtype.categories) + fields = ("year", "month", "day", "hour", "minute", "second") + found_fields = sum(1 for field in fields if getattr(result, field) is not None) + return found_fields >= 2 -def _is_convertible_to_dt(column: pd.Series) -> bool: - try: - var = pd.to_datetime(column, utc=True) - return is_datetime(var) - except Exception: +def _is_convertible_to_num(s: "nw.Series") -> bool: + values = s.drop_nulls().to_list() + if len(values) == 0: return False - - -def _is_convertible_to_num(column: pd.Series) -> bool: try: - ser = pd.to_numeric(column) + for value in values[:100]: + float(value) except (ValueError, TypeError): - ser = column - return is_numeric(ser) + return False + return True -def _is_categorical_and_is_datetime(column: pd.Series) -> bool: - # check for datetime only if the type of the categories is not numeric - # because pd.to_datetime throws an error when it is an integer - if isinstance(column.dtype, pd.CategoricalDtype): - is_dt = not _is_categories_num(column) and _is_convertible_to_dt(column) +def _is_convertible_to_dt(s: "nw.Series") -> bool: + values = s.drop_nulls() + values_list = values.to_list() + if len(values_list) == 0: + return False + + first_value = values_list[0] + if not isinstance(first_value, (date, datetime)): + if _looks_like_date_string(first_value) is False: + return False + + # Try the backend's own vectorized parser first (faster). + # Fall back to the per-value check (below) when it fails. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + try: + values.str.to_datetime() + return True + except Exception: + pass + + for value in values_list[:100]: + if isinstance(value, (date, datetime)): + continue + if _looks_like_date_string(value) is False: + return False + return True + + +def _is_categories_num(s: "nw.Series") -> bool: + return s.cat.get_categories().dtype.is_numeric() + + +def _is_categorical_and_is_not_datetime(s: "nw.Series") -> bool: + if isinstance(s.dtype, nw.Enum): + # an explicit, user-defined category set is an unambiguous categorical + # signal, unlike a generic string column, so skip the datetime check + return True + + if isinstance(s.dtype, nw.Categorical): + # check for datetime only if the categories are not numeric, because + # a numeric-backed categorical (pandas-only - polars categories are + # always string-backed) can never hold dates + categories_are_numeric = _is_categories_num(s) + is_convertible_to_dt = _is_convertible_to_dt(s) + return categories_are_numeric is True or is_convertible_to_dt is False + + if isinstance(s.dtype, (nw.String, nw.Object)): + # check for datetime only if the column cannot be cast as numeric, + # because if it could, it would be a numeric column, not a date + is_convertible_to_num = _is_convertible_to_num(s) + is_convertible_to_dt = _is_convertible_to_dt(s) + return is_convertible_to_num is True or is_convertible_to_dt is False + + return False + + +def _is_categorical_and_is_datetime(s: "nw.Series") -> bool: + if isinstance(s.dtype, nw.Enum): + return False - # check for datetime only if object cannot be cast as numeric because - # if it could pd.to_datetime would convert it to datetime regardless - elif is_object(column): - is_dt = not _is_convertible_to_num(column) and _is_convertible_to_dt(column) + if isinstance(s.dtype, nw.Categorical): + categories_are_numeric = _is_categories_num(s) + is_convertible_to_dt = _is_convertible_to_dt(s) + return categories_are_numeric is False and is_convertible_to_dt is True - else: - is_dt = False + if isinstance(s.dtype, (nw.String, nw.Object)): + is_convertible_to_num = _is_convertible_to_num(s) + is_convertible_to_dt = _is_convertible_to_dt(s) + return is_convertible_to_num is False and is_convertible_to_dt is True - return is_dt + return False diff --git a/feature_engine/variable_handling/check_variables.py b/feature_engine/variable_handling/check_variables.py index 76c4ea7c3..dbff2b434 100644 --- a/feature_engine/variable_handling/check_variables.py +++ b/feature_engine/variable_handling/check_variables.py @@ -2,19 +2,19 @@ from typing import List, Union -import pandas as pd -from pandas.api.types import is_numeric_dtype as is_numeric +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame from feature_engine.variable_handling._variable_type_checks import ( _is_categorical_and_is_datetime, ) -from feature_engine.variable_handling.dtypes import DATETIME_TYPES Variables = Union[int, str, List[Union[str, int]]] def check_numerical_variables( - X: pd.DataFrame, variables: Variables + X: IntoDataFrame, variables: Variables ) -> List[Union[str, int]]: """ Checks that the variables in the list are of type numerical. @@ -23,8 +23,9 @@ def check_numerical_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables : List The list with the names of the variables to check. @@ -41,7 +42,7 @@ def check_numerical_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_ = check_numerical_variables(X, variables=["var_num"]) >>> var_ @@ -51,7 +52,15 @@ def check_numerical_variables( if isinstance(variables, (str, int)): variables = [variables] - if len(X[variables].select_dtypes(exclude="number").columns) > 0: + if nwd.is_pandas_dataframe(X) is True: + not_numerical = len(X[variables].select_dtypes(exclude="number").columns) > 0 + else: + sub_X = nw.from_native(X, eager_only=True).select(variables) + not_numerical = len(sub_X.select(nw.selectors.numeric()).columns) != len( + sub_X.columns + ) + + if not_numerical is True: raise TypeError( "Some of the variables are not numerical. Please cast them as " "numerical before using this transformer." @@ -61,7 +70,7 @@ def check_numerical_variables( def check_categorical_variables( - X: pd.DataFrame, variables: Variables + X: IntoDataFrame, variables: Variables ) -> List[Union[str, int]]: """ Checks that the variables in the list are of type object or categorical. @@ -70,8 +79,9 @@ def check_categorical_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables : list The list with the names of the variables to check. @@ -81,6 +91,13 @@ def check_categorical_variables( variables: List The names of the categorical variables. + Notes + ----- + For polars (and other non-pandas dataframes), plain string columns are + accepted as categorical. Polars has no separate "object" dtype the way + pandas does, so its `String` dtype is the only way to represent free-form + text and is treated as categorical here. + Examples -------- >>> import pandas as pd @@ -88,7 +105,7 @@ def check_categorical_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_ = check_categorical_variables(X, "var_cat") >>> var_ @@ -98,7 +115,24 @@ def check_categorical_variables( if isinstance(variables, (str, int)): variables = [variables] - if len(X[variables].select_dtypes(exclude=["O", "category"]).columns) > 0: + if nwd.is_pandas_dataframe(X) is True: + not_categorical = ( + len( + X[variables] + .select_dtypes(exclude=["O", "category", "string"]) + .columns + ) + > 0 + ) + else: + sub_X = nw.from_native(X, eager_only=True).select(variables) + not_categorical = len( + sub_X.select( + nw.selectors.categorical() | nw.selectors.enum() | nw.selectors.string() + ).columns + ) != len(sub_X.columns) + + if not_categorical is True: raise TypeError( "Some of the variables are not categorical. Please cast them as " "object or categorical before using this transformer." @@ -108,7 +142,7 @@ def check_categorical_variables( def check_datetime_variables( - X: pd.DataFrame, + X: IntoDataFrame, variables: Variables, ) -> List[Union[str, int]]: """ @@ -119,8 +153,9 @@ def check_datetime_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables : list The list with the names of the variables to check. @@ -130,6 +165,12 @@ def check_datetime_variables( variables: List The names of the datetime variables. + Notes + ----- + String columns are parsed with flexible, dateutil-backed date guessing, in + addition to ISO-8601 strings and native `Date`/`Datetime` columns, + regardless of the dataframe library backing `X`. + Examples -------- >>> import pandas as pd @@ -137,7 +178,7 @@ def check_datetime_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_date = check_datetime_variables(X, "var_date") >>> var_date @@ -147,13 +188,27 @@ def check_datetime_variables( if isinstance(variables, (str, int)): variables = [variables] - # find non datetime variables, if any: - non_datetime_vars = [] - for column in X[variables].select_dtypes(exclude=DATETIME_TYPES): - if is_numeric(X[column]) or not _is_categorical_and_is_datetime(X[column]): - non_datetime_vars.append(column) + if nwd.is_pandas_dataframe(X) is True: + sub_X = X[variables] + candidates = sub_X.select_dtypes(exclude=["datetime", "datetimetz"]).columns + numeric_cols = set(sub_X.select_dtypes(include="number").columns) + nw_X = nw.from_native(sub_X, eager_only=True) + non_datetime = any( + column in numeric_cols + or not _is_categorical_and_is_datetime(nw_X.get_column(column)) + for column in candidates + ) + else: + sub_X = nw.from_native(X, eager_only=True).select(variables) + candidates = sub_X.select(~nw.selectors.by_dtype(nw.Date, nw.Datetime)).columns + numeric_cols = set(sub_X.select(nw.selectors.numeric()).columns) + non_datetime = any( + column in numeric_cols + or not _is_categorical_and_is_datetime(sub_X.get_column(column)) + for column in candidates + ) - if len(non_datetime_vars) > 0: + if non_datetime is True: raise TypeError( "Some of the variables are not or cannot be parsed as datetime." ) @@ -162,7 +217,7 @@ def check_datetime_variables( def check_all_variables( - X: pd.DataFrame, + X: IntoDataFrame, variables: Variables, ) -> List[Union[str, int]]: """ @@ -172,8 +227,9 @@ def check_all_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables : list The list with the names of the variables to check. @@ -190,19 +246,24 @@ def check_all_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> vars_all = check_all_variables(X, ['var_num', 'var_cat', 'var_date']) >>> vars_all ['var_num', 'var_cat', 'var_date'] """ + if nwd.is_pandas_dataframe(X) is True: + columns = set(X.columns) + else: + columns = set(nw.from_native(X, eager_only=True).columns) + if isinstance(variables, (str, int)): - if variables not in X.columns.to_list(): + if variables not in columns: raise KeyError(f"The variable {variables} is not in the dataframe.") variables_ = [variables] else: - if not set(variables).issubset(set(X.columns)): + if set(variables).issubset(columns) is False: raise KeyError("Some of the variables are not in the dataframe.") variables_ = variables diff --git a/feature_engine/variable_handling/dtypes.py b/feature_engine/variable_handling/dtypes.py deleted file mode 100644 index c7d93950c..000000000 --- a/feature_engine/variable_handling/dtypes.py +++ /dev/null @@ -1 +0,0 @@ -DATETIME_TYPES = ("datetimetz", "datetime") diff --git a/feature_engine/variable_handling/find_variables.py b/feature_engine/variable_handling/find_variables.py index 5d072eb56..1a19dfaf5 100644 --- a/feature_engine/variable_handling/find_variables.py +++ b/feature_engine/variable_handling/find_variables.py @@ -1,21 +1,55 @@ -"""Functions to select certain types of variables.""" +"""Functions to select different types of variables.""" import warnings -from typing import List, Tuple, Union +from typing import List, Optional, Tuple, Union -import pandas as pd -from pandas.api.types import is_datetime64_any_dtype as is_datetime -from pandas.core.dtypes.common import is_numeric_dtype as is_numeric +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame from feature_engine.variable_handling._variable_type_checks import ( _is_categorical_and_is_datetime, _is_categorical_and_is_not_datetime, ) -from feature_engine.variable_handling.dtypes import DATETIME_TYPES + + +def _find_nw_categoricals( + X: IntoDataFrame, + variables: Optional[List[Union[str, int]]] = None, + exclude_datetime: bool = True, +) -> List[Union[str, int]]: + if nwd.is_pandas_dataframe(X) is True: + sub_X = X if variables is None else X[variables] + candidates = list( + sub_X.select_dtypes(include=["object", "category", "string"]).columns + ) + nw_X = nw.from_native(sub_X, eager_only=True) + else: + nw_X = nw.from_native(X, eager_only=True) + if variables is not None: + nw_X = nw_X.select(variables) + _NW_SELECTOR = ( + nw.selectors.categorical() + | nw.selectors.enum() + | nw.selectors.string() + | nw.selectors.by_dtype(nw.Object) + ) + # `|`-combined selectors don't preserve column order, + # so re-filter over nw_X.columns to restore it. + matched = set(nw_X.select(_NW_SELECTOR).columns) + candidates = [column for column in nw_X.columns if column in matched] + + if exclude_datetime is True: + candidates = [ + column + for column in candidates + if _is_categorical_and_is_not_datetime(nw_X.get_column(column)) + ] + return candidates def find_numerical_variables( - X: pd.DataFrame, + X: IntoDataFrame, return_empty: bool = False, ) -> List[Union[str, int]]: """ @@ -25,8 +59,9 @@ def find_numerical_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. return_empty : bool, default=False Whether to return an empty list when no numerical variables are found. @@ -50,13 +85,18 @@ def find_numerical_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_ = find_numerical_variables(X) >>> var_ ['var_num'] """ - variables = list(X.select_dtypes(include="number").columns) + if nwd.is_pandas_dataframe(X) is True: + variables = list(X.select_dtypes(include="number").columns) + else: + nw_X = nw.from_native(X, eager_only=True) + variables = nw_X.select(nw.selectors.numeric()).columns + if len(variables) == 0: if return_empty is False: raise TypeError( @@ -73,8 +113,9 @@ def find_numerical_variables( def find_categorical_variables( - X: pd.DataFrame, + X: IntoDataFrame, return_empty: bool = False, + exclude_datetime: bool = True, ) -> List[Union[str, int]]: """ Returns a list with the names of all the categorical variables in a dataframe. @@ -85,8 +126,9 @@ def find_categorical_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. return_empty : bool, default=False Whether to return an empty list when no categorical variables are found. @@ -98,6 +140,9 @@ def find_categorical_variables( warning, explicitly set `return_empty=False` instead of relying on the default. + exclude_datetime: bool, default=True + Whether to exclude variables that can be parsed as datetime. + Returns ------- variables: List @@ -110,17 +155,14 @@ def find_categorical_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_ = find_categorical_variables(X) >>> var_ ['var_cat'] """ - variables = [ - column - for column in X.select_dtypes(include=["O", "category", "string"]).columns - if _is_categorical_and_is_not_datetime(X[column]) - ] + variables = _find_nw_categoricals(X, exclude_datetime=exclude_datetime) + if len(variables) == 0: if return_empty is False: raise TypeError( @@ -138,7 +180,7 @@ def find_categorical_variables( def find_datetime_variables( - X: pd.DataFrame, + X: IntoDataFrame, return_empty: bool = False, ) -> List[Union[str, int]]: """ @@ -152,8 +194,9 @@ def find_datetime_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. return_empty : bool, default=False Whether to return an empty list when no datetime variables are found. @@ -170,6 +213,13 @@ def find_datetime_variables( variables: List The names of the datetime variables. + Notes + ----- + String columns are parsed with flexible, dateutil-backed date guessing, so + formats like "01-Jan-2010" or "10/11/12" are recognised, in addition to + ISO-8601 strings and native `Date`/`Datetime` columns, regardless of the + dataframe library backing `X`. + Examples -------- >>> import pandas as pd @@ -177,18 +227,30 @@ def find_datetime_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_date = find_datetime_variables(X) >>> var_date ['var_date'] """ + if nwd.is_pandas_dataframe(X) is True: + non_numeric = X.select_dtypes(exclude="number").columns + datetime_cols = set(X.select_dtypes(include=["datetime", "datetimetz"]).columns) + nw_X = nw.from_native(X, eager_only=True) + else: + nw_X = nw.from_native(X, eager_only=True) + non_numeric = nw_X.select(~nw.selectors.numeric()).columns + datetime_cols = set( + nw_X.select(nw.selectors.by_dtype(nw.Date, nw.Datetime)).columns + ) variables = [ column - for column in X.select_dtypes(exclude="number").columns - if is_datetime(X[column]) or _is_categorical_and_is_datetime(X[column]) + for column in non_numeric + if column in datetime_cols + or _is_categorical_and_is_datetime(nw_X.get_column(column)) ] + if len(variables) == 0: if return_empty is False: raise TypeError( @@ -205,7 +267,7 @@ def find_datetime_variables( def find_all_variables( - X: pd.DataFrame, + X: IntoDataFrame, exclude_datetime: bool = False, return_empty: bool = False, ) -> List[Union[str, int]]: @@ -217,8 +279,9 @@ def find_all_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. exclude_datetime: bool, default=False Whether to exclude datetime variables. @@ -245,21 +308,40 @@ def find_all_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> vars_all = find_all_variables(X) >>> vars_all ['var_num', 'var_cat', 'var_date'] """ - if exclude_datetime is True: - variables = X.select_dtypes(exclude=DATETIME_TYPES).columns.to_list() - variables = [ - var - for var in variables - if is_numeric(X[var]) or not _is_categorical_and_is_datetime(X[var]) - ] + if nwd.is_pandas_dataframe(X) is True: + if exclude_datetime is True: + variables = X.select_dtypes(exclude=["datetime", "datetimetz"]).columns + numeric_cols = set(X.select_dtypes(include="number").columns) + nw_X = nw.from_native(X, eager_only=True) + variables = [ + var + for var in variables + if var in numeric_cols + or not _is_categorical_and_is_datetime(nw_X.get_column(var)) + ] + else: + variables = list(X.columns) else: - variables = X.columns.to_list() + nw_X = nw.from_native(X, eager_only=True) + if exclude_datetime is True: + variables = nw_X.select( + ~nw.selectors.by_dtype(nw.Date, nw.Datetime) + ).columns + numeric_cols = set(nw_X.select(nw.selectors.numeric()).columns) + variables = [ + var + for var in variables + if var in numeric_cols + or not _is_categorical_and_is_datetime(nw_X.get_column(var)) + ] + else: + variables = nw_X.columns if len(variables) == 0: if return_empty is False: @@ -276,9 +358,10 @@ def find_all_variables( def find_categorical_and_numerical_variables( - X: pd.DataFrame, + X: IntoDataFrame, variables: Union[None, int, str, List[Union[str, int]]] = None, return_empty: bool = False, + exclude_datetime: bool = True, ) -> Tuple[List[Union[str, int]], List[Union[str, int]]]: """ Find numerical and categorical variables in a dataframe or from a list. @@ -290,8 +373,9 @@ def find_categorical_and_numerical_variables( Parameters ---------- - X : pandas dataframe of shape = [n_samples, n_features] - The dataset. + X : dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables : list, default=None If `None`, the function finds all categorical and numerical variables in X. @@ -308,6 +392,9 @@ def find_categorical_and_numerical_variables( warning, explicitly set `return_empty=False` instead of relying on the default. + exclude_datetime: bool, default=True + Whether to exclude variables that can be parsed as datetime. + Returns ------- variables: tuple @@ -323,21 +410,28 @@ def find_categorical_and_numerical_variables( >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> var_cat, var_num = find_categorical_and_numerical_variables(X) >>> var_cat, var_num (['var_cat'], ['var_num']) """ + nw_X = nw.from_native(X, eager_only=True) # If the user passes just 1 variable outside a list. if isinstance(variables, (str, int)): - if X[variables].dtype.name == "category" or _is_categorical_and_is_not_datetime( - X[variables] - ): + s = nw_X.get_column(variables) + is_cat = bool( + _find_nw_categoricals( + X, variables=[variables], exclude_datetime=exclude_datetime + ) + ) + is_num = s.dtype.is_numeric() + + if is_cat: variables_cat = [variables] variables_num = [] - elif is_numeric(X[variables]): + elif is_num: variables_num = [variables] variables_cat = [] else: @@ -358,12 +452,11 @@ def find_categorical_and_numerical_variables( # If user leaves default None parameter. elif variables is None: - variables_cat = [ - column - for column in X.select_dtypes(include=["O", "category", "string"]).columns - if _is_categorical_and_is_not_datetime(X[column]) - ] - variables_num = list(X.select_dtypes(include="number").columns) + variables_cat = _find_nw_categoricals(X, exclude_datetime=exclude_datetime) + if nwd.is_pandas_dataframe(X) is True: + variables_num = list(X.select_dtypes(include="number").columns) + else: + variables_num = nw_X.select(nw.selectors.numeric()).columns if len(variables_num) == 0 and len(variables_cat) == 0: if return_empty is False: @@ -399,15 +492,15 @@ def find_categorical_and_numerical_variables( variables_num = [] else: - # find categorical variables - variables_cat = [ - column - for column in X[variables] - .select_dtypes(include=["O", "category", "string"]) - .columns - if _is_categorical_and_is_not_datetime(X[column]) - ] - # find numerical variables - variables_num = list(X[variables].select_dtypes(include="number").columns) + variables_cat = _find_nw_categoricals( + X, variables=variables, exclude_datetime=exclude_datetime + ) + if nwd.is_pandas_dataframe(X) is True: + variables_num = list( + X[variables].select_dtypes(include="number").columns + ) + else: + sub_X = nw_X.select(variables) + variables_num = sub_X.select(nw.selectors.numeric()).columns return variables_cat, variables_num diff --git a/feature_engine/variable_handling/retain_variables.py b/feature_engine/variable_handling/retain_variables.py index 2a161d066..fa3b947ff 100644 --- a/feature_engine/variable_handling/retain_variables.py +++ b/feature_engine/variable_handling/retain_variables.py @@ -2,18 +2,23 @@ from typing import List, Union +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame + Variables = Union[int, str, List[Union[str, int]]] -def retain_variables_if_in_df(X, variables): +def retain_variables_if_in_df(X: IntoDataFrame, variables): """Returns the subset of variables in the list that are present in the dataframe. More details in the :ref:`User Guide `. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] - The dataset. + X: dataframe of shape = [n_samples, n_features] + The dataset. Can be a pandas, polars, or any other dataframe supported by + narwhals. variables: string, int or list of strings or int. The names of the variables to check. @@ -30,7 +35,7 @@ def retain_variables_if_in_df(X, variables): >>> X = pd.DataFrame({ >>> "var_num": [1, 2, 3], >>> "var_cat": ["A", "B", "C"], - >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="T") + >>> "var_date": pd.date_range("2020-02-24", periods=3, freq="min") >>> }) >>> vars_in_df = retain_variables_if_in_df(X, ['var_num', 'var_cat', 'var_other']) >>> vars_in_df @@ -39,7 +44,11 @@ def retain_variables_if_in_df(X, variables): if isinstance(variables, (str, int)): variables = [variables] - variables_in_df = [var for var in variables if var in X.columns] + if nwd.is_pandas_dataframe(X) is True: + columns = set(X.columns) + else: + columns = set(nw.from_native(X, eager_only=True).columns) + variables_in_df = [var for var in variables if var in columns] # Raise an error if no column is left to work with. if len(variables_in_df) == 0: diff --git a/pyproject.toml b/pyproject.toml index 9e5fb0199..64dbb8156 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,18 +10,17 @@ license = {text = "BSD 3 clause"} authors = [ { name = "Soledad Galli", email = "solegalli@protonmail.com" } ] -requires-python = ">=3.9.0" +requires-python = ">=3.11.0" dependencies = [ "numpy>=1.18.2", - "pandas>=2.2.0", - "scikit-learn>=1.4.0", + "scikit-learn>=1.7.0", "scipy>=1.4.1", + "narwhals>=2.0.0", + "python-dateutil>=2.8.2", ] classifiers = [ "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -39,10 +38,13 @@ docs = [ "pydata_sphinx_theme>=0.7.2", "sphinx_autodoc_typehints>=1.11.1,<=1.21.3", "numpydoc>=0.9.2", + "pandas>=2.2.0", ] tests = [ "pytest>=5.4.1", + "pandas>=2.2.0", + "polars>=1.0.0", # repo maintenance tooling "black>=21.5b1", diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6df436a29..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy>=1.18.2 -pandas>=2.2.0 -scikit-learn>=1.4.0 -scipy>=1.4.1 diff --git a/tests/check_estimators_with_parametrize_tests.py b/tests/check_estimators_with_parametrize_tests.py deleted file mode 100644 index 039bd50c2..000000000 --- a/tests/check_estimators_with_parametrize_tests.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -This file is only intended to help understand check_estimator tests on Feature-engine -transformers. It is not run as part of the battery of acceptance tests. Works up to -sklearn < 1.6. -""" - -from sklearn.impute import SimpleImputer -from sklearn.linear_model import LogisticRegression -from sklearn.utils.estimator_checks import parametrize_with_checks - -from feature_engine.creation import ( - CyclicalFeatures, - DecisionTreeFeatures, - MathFeatures, - RelativeFeatures, -) -from feature_engine.encoding import ( - CountEncoder, - DecisionTreeEncoder, - MeanEncoder, - OneHotEncoder, - OrdinalEncoder, - RareLabelEncoder, - StringSimilarityEncoder, - WoEEncoder, -) -from feature_engine.imputation import ( - AddMissingIndicator, - ArbitraryImputer, - CategoricalImputer, - DropMissingData, - EndTailImputer, - MeanImputer, - RandomSampleImputer, -) -from feature_engine.outliers import ArbitraryOutlierCapper, OutlierTrimmer, Winsoriser -from feature_engine.selection import ( - MRMR, - DropConstantFeatures, - DropCorrelatedFeatures, - DropDuplicateFeatures, - DropFeatures, - DropHighPSIFeatures, - ProbeFeatureSelection, - RecursiveFeatureAddition, - RecursiveFeatureElimination, - SelectByInformationValue, - SelectByShuffling, - SelectBySingleFeaturePerformance, - SelectByTargetEncoding, - SmartCorrelatedSelection, -) -from feature_engine.timeseries.forecasting import ( - ExpandingWindowFeatures, - LagFeatures, - WindowFeatures, -) -from feature_engine.transformation import ( - ArcsinTransformer, - BoxCoxTransformer, - LogTransformer, - PowerTransformer, - ReciprocalTransformer, - YeoJohnsonTransformer, -) -from feature_engine.wrappers import SklearnWrapper - - -# creation -@parametrize_with_checks( - [ - DecisionTreeFeatures(regression=False), - CyclicalFeatures(), - MathFeatures(variables=["x0", "x1"], func="mean", missing_values="ignore"), - RelativeFeatures( - variables=["x0", "x1"], - reference=["x0"], - func=["add"], - missing_values="ignore", - ), - ] -) -def test_sklearn_compatible_creator(estimator, check): - check(estimator) - - -# imputation -@parametrize_with_checks( - [ - MeanImputer(), - ArbitraryImputer(), - CategoricalImputer(fill_value=0, ignore_format=True), - EndTailImputer(), - AddMissingIndicator(), - RandomSampleImputer(), - DropMissingData(), - ] -) -def test_sklearn_compatible_imputer(estimator, check): - check(estimator) - - -# encoding -@parametrize_with_checks( - [ - CountEncoder(ignore_format=True), - DecisionTreeEncoder(regression=False, ignore_format=True), - MeanEncoder(ignore_format=True), - OneHotEncoder(ignore_format=True), - OrdinalEncoder(ignore_format=True), - RareLabelEncoder( - tol=0.00000000001, - n_categories=100000000000, - replace_with=10, - ignore_format=True, - ), - WoEEncoder(ignore_format=True), - StringSimilarityEncoder(ignore_format=True), - ] -) -def test_sklearn_compatible_encoder(estimator, check): - check(estimator) - - -# outliers -@parametrize_with_checks( - [ - ArbitraryOutlierCapper(max_capping_dict={"x0": 10}), - OutlierTrimmer(), - Winsoriser(), - ] -) -def test_sklearn_compatible_outliers(estimator, check): - check(estimator) - - -# transformers -@parametrize_with_checks( - [ - ArcsinTransformer(), - BoxCoxTransformer(), - LogTransformer(), - PowerTransformer(), - ReciprocalTransformer(), - YeoJohnsonTransformer(), - ] -) -def test_sklearn_compatible_transformer(estimator, check): - check(estimator) - - -# selectors -@parametrize_with_checks( - [ - DropFeatures(features_to_drop=["x0"]), - DropConstantFeatures(missing_values="ignore"), - DropDuplicateFeatures(), - DropCorrelatedFeatures(), - SmartCorrelatedSelection(), - DropHighPSIFeatures(bins=5), - SelectByShuffling( - LogisticRegression(max_iter=2, random_state=1), scoring="accuracy" - ), - SelectBySingleFeaturePerformance( - LogisticRegression(max_iter=2, random_state=1), scoring="accuracy" - ), - RecursiveFeatureAddition( - LogisticRegression(max_iter=2, random_state=1), scoring="accuracy" - ), - RecursiveFeatureElimination( - LogisticRegression(max_iter=2, random_state=1), - scoring="accuracy", - threshold=-100, - ), - SelectByTargetEncoding(scoring="roc_auc", bins=3, regression=False), - SelectByInformationValue(), - MRMR(), - ProbeFeatureSelection(estimator=LogisticRegression()), - ] -) -def test_sklearn_compatible_selectors(estimator, check): - check(estimator) - - -# wrappers -@parametrize_with_checks([SklearnWrapper(SimpleImputer())]) -def test_sklearn_compatible_wrapper(estimator, check): - check(estimator) - - -# test_forecasting -@parametrize_with_checks( - [ - LagFeatures(missing_values="ignore"), - WindowFeatures(missing_values="ignore"), - ExpandingWindowFeatures(missing_values="ignore"), - ] -) -def test_sklearn_compatible_forecasters(estimator, check): - check(estimator) diff --git a/tests/test_base_transformers/test_get_feature_names_out_mixin.py b/tests/test_base_transformers/test_get_feature_names_out_mixin.py index e4b67ed33..7315b694b 100644 --- a/tests/test_base_transformers/test_get_feature_names_out_mixin.py +++ b/tests/test_base_transformers/test_get_feature_names_out_mixin.py @@ -1,4 +1,6 @@ import numpy as np +import pandas as pd +import polars as pl import pytest from sklearn.base import BaseEstimator from sklearn.exceptions import NotFittedError @@ -9,9 +11,14 @@ from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin from feature_engine.dataframe_checks import check_X -variables_str = ["Name", "City", "Age", "Marks", "dob"] -variables_arr = ["x0", "x1", "x2", "x3", "x4"] -variables_user = ["Dog", "Cat", "Bird", "Frog", "Duck"] +VARTYPES_DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": ["2020-02-24", "2020-02-25", "2020-02-26", "2020-02-27"], +} +variables_str = list(VARTYPES_DATA.keys()) class MockTransformer(BaseEstimator, GetFeatureNamesOutMixin): @@ -25,40 +32,45 @@ def transform(self, X): return X.copy() -def test_non_fitted_error(df_vartypes): +def test_non_fitted_error(): transformer = MockTransformer() with pytest.raises(NotFittedError): - transformer.get_feature_names_out(df_vartypes) + transformer.get_feature_names_out() # ======== Tests for transformers that do not add new features to the data ======== -def test_when_input_is_pandas_columns(df_vartypes): - input_features = df_vartypes.columns +def test_when_input_is_pandas_columns(): + df = pd.DataFrame(VARTYPES_DATA) transformer = MockTransformer() - - transformer.fit(df_vartypes) + transformer.fit(df) assert ( - transformer.get_feature_names_out(input_features=input_features) - == variables_str + transformer.get_feature_names_out(input_features=df.columns) == variables_str ) - transformer.fit(df_vartypes.to_numpy()) + +def test_when_input_is_polars_columns(): + # polars' .columns is already a plain list, so this exercises the + # `isinstance(input_features, list)` branch, not `nwd.is_pandas_index`. + df = pl.DataFrame(VARTYPES_DATA) + transformer = MockTransformer() + transformer.fit(df) assert ( - transformer.get_feature_names_out(input_features=input_features) - == variables_str + transformer.get_feature_names_out(input_features=df.columns) == variables_str ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_with_df(df_vartypes, input_features): +def test_with_df(make_df, input_features): # When the data used to train the class is a dataframe, the variable names are # stored in feature_names_in_. Those should be returned by get_feature_names_out() + df = make_df(VARTYPES_DATA) transformer = MockTransformer() - transformer.fit(df_vartypes) + transformer.fit(df) assert ( transformer.get_feature_names_out(input_features=input_features) == transformer.feature_names_in_ @@ -67,48 +79,16 @@ def test_with_df(df_vartypes, input_features): transformer.get_feature_names_out(input_features=input_features) == variables_str ) - assert ( - transformer.get_feature_names_out(input_features=df_vartypes.columns) - == variables_str - ) - - -@pytest.mark.parametrize( - "input_features", - [ - None, - variables_arr, - np.array(variables_arr), - variables_str, - np.array(variables_str), - variables_user, - ], -) -def test_with_array(df_vartypes, input_features): - # When the data used to train the class is a numpy array, the names stored in - # feature_names_in_ are x0, x1, etc. Those should be returned by - # get_feature_names_out() when input_features is None. Alternatively, it returns - # a list of the variables entered by the user. - transformer = MockTransformer() - transformer.fit(df_vartypes.to_numpy()) - - if input_features is None: - assert ( - transformer.get_feature_names_out(input_features=input_features) - == variables_arr - ) - else: - assert transformer.get_feature_names_out(input_features=input_features) == list( - input_features - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_with_pipeline_and_df(df_vartypes, input_features): +def test_with_pipeline_and_df(make_df, input_features): + df = make_df(VARTYPES_DATA) pipe = Pipeline([("transformer", MockTransformer())]) - pipe.fit(df_vartypes) + pipe.fit(df) assert ( pipe.get_feature_names_out(input_features=input_features) == pipe.named_steps["transformer"].feature_names_in_ @@ -116,113 +96,35 @@ def test_with_pipeline_and_df(df_vartypes, input_features): assert pipe.get_feature_names_out(input_features=input_features) == variables_str -@pytest.mark.parametrize( - "input_features", - [ - None, - variables_arr, - np.array(variables_arr), - variables_str, - np.array(variables_str), - variables_user, - ], -) -def test_with_pipeline_and_array(df_vartypes, input_features): - pipe = Pipeline([("transformer", MockTransformer())]) - pipe.fit(df_vartypes.to_numpy()) - - if input_features is None: - assert ( - pipe.get_feature_names_out(input_features=input_features) == variables_arr - ) - else: - assert pipe.get_feature_names_out(input_features=input_features) == list( - input_features - ) - - @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_with_pipe_and_skl_transformer_input_df(df_vartypes, input_features): +def test_with_pipe_and_skl_transformer_input_df(input_features): + # SimpleImputer outputs a numpy array by default, which check_X now + # rejects, so it must be configured to output a dataframe. + df = pd.DataFrame(VARTYPES_DATA) pipe = Pipeline( [ - ("imputer", SimpleImputer(strategy="constant")), + ( + "imputer", + SimpleImputer(strategy="constant").set_output(transform="pandas"), + ), ("transformer", MockTransformer()), ] ) - pipe.fit(df_vartypes) + pipe.fit(df) assert pipe.get_feature_names_out(input_features=input_features) == variables_str -@pytest.mark.parametrize( - "input_features", - [ - None, - variables_arr, - np.array(variables_arr), - variables_str, - np.array(variables_str), - variables_user, - ], -) -def test_with_pipe_and_skl_transformer_input_array(df_vartypes, input_features): +def test_pipe_with_skl_transformer_that_adds_features(): + df = pd.DataFrame({"Age": VARTYPES_DATA["Age"], "Marks": VARTYPES_DATA["Marks"]}) pipe = Pipeline( [ - ("imputer", SimpleImputer(strategy="constant")), + ("poly", PolynomialFeatures().set_output(transform="pandas")), ("transformer", MockTransformer()), ] ) - pipe.fit(df_vartypes.to_numpy()) - - if input_features is None: - assert ( - pipe.get_feature_names_out(input_features=input_features) == variables_arr - ) - else: - assert pipe.get_feature_names_out(input_features=input_features) == list( - input_features - ) - - -def test_pipe_with_skl_transformer_that_adds_features(df_vartypes): - pipe = Pipeline( - [ - ("poly", PolynomialFeatures()), - ("transformer", MockTransformer()), - ] - ) - - # when input is array - pipe.fit(df_vartypes[["Age", "Marks"]].to_numpy()) - assert pipe.get_feature_names_out(input_features=None) == [ - "1", - "x0", - "x1", - "x0^2", - "x0 x1", - "x1^2", - ] - - assert pipe.get_feature_names_out(input_features=["Age", "Marks"]) == [ - "1", - "Age", - "Marks", - "Age^2", - "Age Marks", - "Marks^2", - ] - assert pipe.get_feature_names_out(input_features=["Dog", "Cat"]) == [ - "1", - "Dog", - "Cat", - "Dog^2", - "Dog Cat", - "Cat^2", - ] - - # when input is df - pipe.fit(df_vartypes[["Age", "Marks"]]) + pipe.fit(df) assert pipe.get_feature_names_out(input_features=None) == [ "1", "Age", @@ -242,32 +144,22 @@ def test_pipe_with_skl_transformer_that_adds_features(df_vartypes): ] -def test_raise_error_when_input_feature_non_permitted(df_vartypes): +def test_raise_error_when_input_feature_non_permitted(): + df = pd.DataFrame(VARTYPES_DATA) transformer = MockTransformer() + transformer.fit(df) - # when input is dataframe - transformer.fit(df_vartypes) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match="feature_names_in_"): transformer.get_feature_names_out(input_features=["Name"]) - assert "feature_names_in_" in str(record) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match="feature_names_in_"): transformer.get_feature_names_out(input_features=np.array(["Name", "Age"])) - assert "feature_names_in_" in str(record) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match="list or an array"): transformer.get_feature_names_out(input_features="var1") - assert "list or an array" in str(record) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match="list or an array"): transformer.get_feature_names_out(input_features=True) - assert "list or an array" in str(record) - - # when input is array - transformer.fit(df_vartypes.to_numpy()) - with pytest.raises(ValueError) as record: - transformer.get_feature_names_out(input_features=["Name", "Age"]) - assert "number of input_features does not match" in str(record) # ================ Tests for transformers that add features to the data ======= @@ -292,21 +184,23 @@ def _get_new_features_name(self): return [f"{i}_plus" for i in self.variables_] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("features_in", [["Age", "Marks"], ["Name", "dob"]]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_new_feature_names_with_df(df_vartypes, features_in, input_features): +def test_new_feature_names_with_df(make_df, features_in, input_features): + df = make_df(VARTYPES_DATA) transformer = MockCreator(variables=features_in, drop_original=False) - transformer.fit(df_vartypes) - features_out = list(df_vartypes.columns) + [f"{i}_plus" for i in features_in] + transformer.fit(df) + features_out = variables_str + [f"{i}_plus" for i in features_in] assert ( transformer.get_feature_names_out(input_features=input_features) == features_out ) transformer = MockCreator(variables=features_in, drop_original=True) - transformer.fit(df_vartypes) - features_out = [f for f in df_vartypes.columns if f not in features_in] + [ + transformer.fit(df) + features_out = [f for f in variables_str if f not in features_in] + [ f"{i}_plus" for i in features_in ] assert ( @@ -314,18 +208,20 @@ def test_new_feature_names_with_df(df_vartypes, features_in, input_features): ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("features_in", [["Age", "Marks"], ["Name", "dob"]]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_new_feature_names_within_pipeline(df_vartypes, features_in, input_features): +def test_new_feature_names_within_pipeline(make_df, features_in, input_features): + df = make_df(VARTYPES_DATA) transformer = Pipeline( [ ("transformer", MockCreator(variables=features_in, drop_original=False)), ] ) - transformer.fit(df_vartypes) - features_out = list(df_vartypes.columns) + [f"{i}_plus" for i in features_in] + transformer.fit(df) + features_out = variables_str + [f"{i}_plus" for i in features_in] assert ( transformer.get_feature_names_out(input_features=input_features) == features_out ) @@ -335,8 +231,8 @@ def test_new_feature_names_within_pipeline(df_vartypes, features_in, input_featu ("transformer", MockCreator(variables=features_in, drop_original=True)), ] ) - transformer.fit(df_vartypes) - features_out = [f for f in df_vartypes.columns if f not in features_in] + [ + transformer.fit(df) + features_out = [f for f in variables_str if f not in features_in] + [ f"{i}_plus" for i in features_in ] assert ( @@ -349,25 +245,33 @@ def test_new_feature_names_within_pipeline(df_vartypes, features_in, input_featu "input_features", [None, variables_str, np.array(variables_str)] ) def test_new_feature_names_pipe_with_skl_transformer_and_df( - df_vartypes, features_in, input_features + features_in, input_features ): + df = pd.DataFrame(VARTYPES_DATA) pipe = Pipeline( [ - ("imputer", SimpleImputer(strategy="constant")), + ( + "imputer", + SimpleImputer(strategy="constant").set_output(transform="pandas"), + ), ("transformer", MockCreator(variables=features_in, drop_original=False)), ] ) - pipe.fit(df_vartypes) - features_out = list(df_vartypes.columns) + [f"{i}_plus" for i in features_in] + pipe.fit(df) + features_out = variables_str + [f"{i}_plus" for i in features_in] assert pipe.get_feature_names_out(input_features=input_features) == features_out + pipe = Pipeline( [ - ("imputer", SimpleImputer(strategy="constant")), + ( + "imputer", + SimpleImputer(strategy="constant").set_output(transform="pandas"), + ), ("transformer", MockCreator(variables=features_in, drop_original=True)), ] ) - pipe.fit(df_vartypes) - features_out = [f for f in df_vartypes.columns if f not in features_in] + [ + pipe.fit(df) + features_out = [f for f in variables_str if f not in features_in] + [ f"{i}_plus" for i in features_in ] assert pipe.get_feature_names_out(input_features=input_features) == features_out @@ -376,15 +280,13 @@ def test_new_feature_names_pipe_with_skl_transformer_and_df( @pytest.mark.parametrize( "input_features", [None, ["Age", "Marks"], np.array(["Age", "Marks"])] ) -def test_new_feature_names_pipe_and_skl_transformer_that_adds_features( - df_vartypes, input_features -): +def test_new_feature_names_pipe_and_skl_transformer_that_adds_features(input_features): features_in = ["Age", "Marks"] - df = df_vartypes[features_in].copy() + df = pd.DataFrame({"Age": VARTYPES_DATA["Age"], "Marks": VARTYPES_DATA["Marks"]}) pipe = Pipeline( [ - ("poly", PolynomialFeatures()), + ("poly", PolynomialFeatures().set_output(transform="pandas")), ("transformer", MockCreator(variables=features_in, drop_original=False)), ] ) @@ -419,41 +321,29 @@ def get_support(self, indices=False): return mask if not indices else np.where(mask)[0] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_remove_features_in_df(df_vartypes, input_features): +def test_remove_features_in_df(make_df, input_features): + df = make_df(VARTYPES_DATA) transformer = MockSelector() - transformer.fit(df_vartypes) - features_out = list(df_vartypes.columns)[2:] - assert ( - transformer.get_feature_names_out(input_features=input_features) == features_out - ) - - -@pytest.mark.parametrize( - "input_features", - [None, variables_arr, np.array(variables_arr), variables_str, variables_user], -) -def test_remove_features_in_array(df_vartypes, input_features): - transformer = MockSelector() - transformer.fit(df_vartypes.to_numpy()) - if input_features is None: - features_out = ["x2", "x3", "x4"] - else: - features_out = list(input_features)[2:] + transformer.fit(df) + features_out = variables_str[2:] assert ( transformer.get_feature_names_out(input_features=input_features) == features_out ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_remove_feature_names_within_pipeline_when_df(df_vartypes, input_features): +def test_remove_feature_names_within_pipeline_when_df(make_df, input_features): + df = make_df(VARTYPES_DATA) transformer = Pipeline([("transformer", MockSelector())]) - transformer.fit(df_vartypes) - features_out = list(df_vartypes.columns)[2:] + transformer.fit(df) + features_out = variables_str[2:] assert ( transformer.get_feature_names_out(input_features=input_features) == features_out ) @@ -462,73 +352,60 @@ def test_remove_feature_names_within_pipeline_when_df(df_vartypes, input_feature @pytest.mark.parametrize( "input_features", [None, variables_str, np.array(variables_str)] ) -def test_remove_feature_names_pipe_with_skl_transformer_and_df( - df_vartypes, input_features -): - df_vartypes = df_vartypes.drop(["dob"], axis=1) - if input_features is not None: - input_features = input_features[0:-1] +def test_remove_feature_names_pipe_with_skl_transformer_and_df(input_features): + df = pd.DataFrame( + {k: v for k, v in VARTYPES_DATA.items() if k != "dob"} + ) + variables_no_dob = [v for v in variables_str if v != "dob"] + trimmed_input_features = ( + input_features[0:-1] if input_features is not None else None + ) pipe = Pipeline( [ ("transformer", MockSelector()), - ("imputer", SimpleImputer(strategy="constant")), + ( + "imputer", + SimpleImputer(strategy="constant").set_output(transform="pandas"), + ), ] ) - pipe.fit(df_vartypes) - features_out = list(df_vartypes.columns)[2:] + pipe.fit(df) + features_out = variables_no_dob[2:] + # sklearn's Pipeline.get_feature_names_out() returns a numpy array here + # when the feature-removing transformer isn't the last step. assert all( - pipe.get_feature_names_out(input_features=input_features) == features_out + pipe.get_feature_names_out(input_features=trimmed_input_features) + == features_out ) pipe = Pipeline( [ - ("imputer", SimpleImputer(strategy="constant")), + ( + "imputer", + SimpleImputer(strategy="constant").set_output(transform="pandas"), + ), ("transformer", MockSelector()), ] ) - pipe.fit(df_vartypes) - features_out = list(df_vartypes.columns)[2:] - assert pipe.get_feature_names_out(input_features=input_features) == features_out - - -@pytest.mark.parametrize( - "input_features", [None, variables_str, variables_arr, variables_user] -) -def test_new_feature_names_pipe_with_skl_transformer_and_array( - df_vartypes, input_features -): - df_vartypes = df_vartypes.drop(["dob"], axis=1) - - pipe = Pipeline( - [ - ("imputer", SimpleImputer(strategy="constant")), - ("transformer", MockSelector()), - ] + pipe.fit(df) + assert ( + pipe.get_feature_names_out(input_features=trimmed_input_features) + == features_out ) - pipe.fit(df_vartypes.to_numpy()) - - if input_features is not None: - input_features = input_features[0:-1] - features_out = input_features[2:] - assert pipe.get_feature_names_out(input_features=input_features) == features_out - else: - features_out = ["x2", "x3"] - assert pipe.get_feature_names_out(input_features=input_features) == features_out @pytest.mark.parametrize( "input_features", [None, ["Age", "Marks"], np.array(["Age", "Marks"])] ) def test_remove_feature_names_pipe_and_skl_transformer_that_adds_features( - df_vartypes, input_features + input_features, ): - features_in = ["Age", "Marks"] - df = df_vartypes[features_in].copy() + df = pd.DataFrame({"Age": VARTYPES_DATA["Age"], "Marks": VARTYPES_DATA["Marks"]}) pipe = Pipeline( [ - ("poly", PolynomialFeatures()), + ("poly", PolynomialFeatures().set_output(transform="pandas")), ("transformer", MockSelector()), ] ) diff --git a/tests/test_base_transformers/test_transform_xy_mixin.py b/tests/test_base_transformers/test_transform_xy_mixin.py index 03a34f3d5..0bd6b4d5c 100644 --- a/tests/test_base_transformers/test_transform_xy_mixin.py +++ b/tests/test_base_transformers/test_transform_xy_mixin.py @@ -1,36 +1,60 @@ -import numpy as np +import narwhals as nw import pandas as pd +import polars as pl +import pytest from feature_engine._base_transformers.mixins import TransformXyMixin +BACKENDS = [(pd.DataFrame, pd.Series), (pl.DataFrame, pl.Series)] + class MockTransformer(TransformXyMixin): def transform(self, X): - return X.iloc[1:-1].copy() + # drops rows at positions 2 and 4, backend-agnostic + nw_X = nw.from_native(X, eager_only=True) + keep = [i for i in range(len(nw_X)) if i not in (2, 4)] + return nw_X[keep].to_native() -def test_transform_x_y_method(df_vartypes): - # single target - y = pd.Series(0, index=np.arange(len(df_vartypes))) +@pytest.mark.parametrize("make_df, make_series", BACKENDS) +def test_transform_x_y_single_target(make_df, make_series): + X = make_df({"a": [0, 1, 2, 3, 4, 5], "b": [10, 11, 12, 13, 14, 15]}) + y = make_series([0, 1, 2, 3, 4, 5]) transformer = MockTransformer() - Xt, yt = transformer.transform_x_y(df_vartypes, y) - assert len(Xt) == len(yt) - assert len(Xt) != len(df_vartypes) - assert len(yt) != len(y) - assert (Xt.index == yt.index).all() - assert (Xt.index == [1, 2]).all() + Xt, yt = transformer.transform_x_y(X, y) + + assert len(Xt) == 4 + assert len(yt) == 4 + assert nw.from_native(yt, series_only=True).to_list() == [0, 1, 3, 5] + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_x_y_multioutput_target(make_df): + X = make_df({"a": [0, 1, 2, 3, 4, 5], "b": [10, 11, 12, 13, 14, 15]}) + y = make_df({"t1": [0, 1, 2, 3, 4, 5], "t2": [0, 10, 20, 30, 40, 50]}) + transformer = MockTransformer() + + Xt, yt = transformer.transform_x_y(X, y) + + assert len(Xt) == 4 + assert len(yt) == 4 + nw_yt = nw.from_native(yt, eager_only=True) + assert nw_yt["t1"].to_list() == [0, 1, 3, 5] + assert nw_yt["t2"].to_list() == [0, 10, 30, 50] + + +def test_transform_x_y_pandas_index_alignment(df_vartypes): + # pandas branch keeps the original (non-default) index aligned between X and y + class DropFirstAndLast(TransformXyMixin): + def transform(self, X): + return X.iloc[1:-1].copy() - # multioutput target - y = ( - pd.DataFrame(columns=["vara", "varb"], index=df_vartypes.index) - .astype(float) - .fillna(0) - ) + y = pd.Series(range(len(df_vartypes)), index=df_vartypes.index) + transformer = DropFirstAndLast() Xt, yt = transformer.transform_x_y(df_vartypes, y) assert len(Xt) == len(yt) assert len(Xt) != len(df_vartypes) - assert len(yt) != len(y) assert (Xt.index == yt.index).all() assert (Xt.index == [1, 2]).all() diff --git a/tests/test_creation/test_base_creation.py b/tests/test_creation/test_base_creation.py new file mode 100644 index 000000000..d32976007 --- /dev/null +++ b/tests/test_creation/test_base_creation.py @@ -0,0 +1,122 @@ +import narwhals as nw +import pandas as pd +import polars as pl +import pytest + +from feature_engine.creation.base_creation import BaseCreation + +BASIC_DATA = { + "var_a": [1, 2, 3, 4], + "var_b": [10, 20, 30, 40], + "var_c": [100, 200, 300, 400], +} + + +class StubCreation(BaseCreation): + def __init__(self, variables=None, missing_values="raise", drop_original=False): + self.variables = variables + super().__init__(missing_values=missing_values, drop_original=drop_original) + + def transform(self, X): + return self._check_transform_input_and_state(X) + + +class StubWithReference(StubCreation): + def __init__( + self, reference, variables=None, missing_values="raise", drop_original=False + ): + self.reference = reference + super().__init__( + variables=variables, + missing_values=missing_values, + drop_original=drop_original, + ) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_transform_round_trip(make_df): + X = make_df(BASIC_DATA) + transformer = StubCreation() + transformer.fit(X) + + assert transformer.variables_ == ["var_a", "var_b", "var_c"] + assert transformer.feature_names_in_ == ["var_a", "var_b", "var_c"] + assert transformer.n_features_in_ == 3 + + Xt = transformer.transform(X) + assert list(nw.from_native(Xt, eager_only=True).columns) == [ + "var_a", + "var_b", + "var_c", + ] + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_reorders_columns_to_match_fit(make_df): + X = make_df(BASIC_DATA) + transformer = StubCreation() + transformer.fit(X) + + reordered = make_df( + { + "var_c": BASIC_DATA["var_c"], + "var_a": BASIC_DATA["var_a"], + "var_b": BASIC_DATA["var_b"], + } + ) + Xt = transformer.transform(reordered) + assert list(nw.from_native(Xt, eager_only=True).columns) == [ + "var_a", + "var_b", + "var_c", + ] + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_when_column_count_differs(make_df): + X = make_df(BASIC_DATA) + transformer = StubCreation() + transformer.fit(X) + + X_fewer_cols = make_df( + {"var_a": BASIC_DATA["var_a"], "var_b": BASIC_DATA["var_b"]} + ) + msg = ( + "The number of columns in this dataset is different from the one used to " + "fit this transformer" + ) + with pytest.raises(ValueError, match=msg): + transformer.transform(X_fewer_cols) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_missing_values_raise_vs_ignore(make_df): + data_with_na = {**BASIC_DATA, "var_a": [1, None, 3, 4]} + X = make_df(data_with_na) + + transformer_raise = StubCreation(missing_values="raise") + msg = "Some of the variables in the dataset contain NaN" + with pytest.raises(ValueError, match=msg): + transformer_raise.fit(X) + + transformer_ignore = StubCreation(missing_values="ignore") + transformer_ignore.fit(X) + Xt = transformer_ignore.transform(X) + assert len(nw.from_native(Xt, eager_only=True)) == 4 + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_reference_attribute_is_checked_in_fit(make_df): + X = make_df(BASIC_DATA) + transformer = StubWithReference(reference=["var_a"]) + transformer.fit(X) + assert transformer.variables_ == ["var_a", "var_b", "var_c"] + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_reference_must_be_numerical(make_df): + X = make_df({**BASIC_DATA, "var_d": ["a", "b", "c", "d"]}) + transformer = StubWithReference(reference=["var_d"]) + msg = "Some of the variables are not numerical" + with pytest.raises(TypeError, match=msg): + transformer.fit(X) diff --git a/tests/test_creation/test_check_estimator_creation.py b/tests/test_creation/test_check_estimator_creation.py index 23dec93c3..dc8beaa9e 100644 --- a/tests/test_creation/test_check_estimator_creation.py +++ b/tests/test_creation/test_check_estimator_creation.py @@ -1,9 +1,7 @@ import pandas as pd import pytest -import sklearn from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.creation import ( CyclicalFeatures, @@ -17,8 +15,6 @@ check_raises_non_fitted_error_when_fit_fails, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - # Estimators for sklearn's check_estimator # Note: GeoDistanceFeatures is not included here because it requires 4 specific # named coordinate columns, but sklearn's check_estimator generates test data @@ -32,20 +28,13 @@ DecisionTreeFeatures(regression=False), ] -if sklearn_version > parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator( - estimator=estimator, - expected_failed_checks=estimator._more_tags()["_xfail_checks"], - ) - -else: - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + return check_estimator( + estimator=estimator, + expected_failed_checks=estimator._more_tags()["_xfail_checks"], + ) _estimators = [ diff --git a/tests/test_creation/test_cyclical_features.py b/tests/test_creation/test_cyclical_features.py index 5bc1df88f..ab834346f 100644 --- a/tests/test_creation/test_cyclical_features.py +++ b/tests/test_creation/test_cyclical_features.py @@ -1,29 +1,33 @@ +import narwhals as nw import pandas as pd +import polars as pl import pytest from numpy import array from feature_engine.creation import CyclicalFeatures +CYCLICAL_DATA = { + "day": [6, 7, 5, 3, 1, 2, 4], + "months": [3, 7, 9, 12, 4, 6, 12], +} -@pytest.fixture -def df_cyclical(): - df = { - "day": [6, 7, 5, 3, 1, 2, 4], - "months": [3, 7, 9, 12, 4, 6, 12], - } - df = pd.DataFrame(df) - return df + +def assert_df_equal(X, expected: dict) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert result[col] == pytest.approx(values, abs=1e-5) -def test_general_transformation_without_dropping_variables(df_cyclical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_general_transformation_without_dropping_variables(make_df): # test case 1: just one variable. + df = make_df(CYCLICAL_DATA) cyclical = CyclicalFeatures(variables=["day"]) - X = cyclical.fit_transform(df_cyclical) + X = cyclical.fit_transform(df) - transf_df = df_cyclical.copy() - - # expected output - transf_df["day_sin"] = [ + expected = dict(CYCLICAL_DATA) + expected["day_sin"] = [ -0.78183, 0.0, -0.97493, @@ -32,7 +36,7 @@ def test_general_transformation_without_dropping_variables(df_cyclical): 0.97493, -0.43388, ] - transf_df["day_cos"] = [ + expected["day_cos"] = [ 0.623490, 1.0, -0.222521, @@ -46,18 +50,18 @@ def test_general_transformation_without_dropping_variables(df_cyclical): assert cyclical.max_values_ == {"day": 7} # test transform output - pd.testing.assert_frame_equal(X, transf_df) + assert_df_equal(X, expected) -def test_general_transformation_dropping_original_variables(df_cyclical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_general_transformation_dropping_original_variables(make_df): # test case 1: just one variable, but dropping the variable after transformation + df = make_df(CYCLICAL_DATA) cyclical = CyclicalFeatures(variables=["day"], drop_original=True) - X = cyclical.fit_transform(df_cyclical) - - transf_df = df_cyclical.copy() + X = cyclical.fit_transform(df) - # expected output - transf_df["day_sin"] = [ + expected = dict(CYCLICAL_DATA) + expected["day_sin"] = [ -0.78183, 0.0, -0.97493, @@ -66,7 +70,7 @@ def test_general_transformation_dropping_original_variables(df_cyclical): 0.97493, -0.43388, ] - transf_df["day_cos"] = [ + expected["day_cos"] = [ 0.623490, 1.0, -0.222521, @@ -75,60 +79,61 @@ def test_general_transformation_dropping_original_variables(df_cyclical): -0.222521, -0.900969, ] - transf_df = transf_df.drop(columns="day") + del expected["day"] # test fit attr assert cyclical.n_features_in_ == 2 assert cyclical.max_values_ == {"day": 7} # test transform output - pd.testing.assert_frame_equal(X, transf_df) + assert_df_equal(X, expected) -def test_automatically_find_variables(df_cyclical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_find_variables(make_df): # test case 2: automatically select variables + df = make_df(CYCLICAL_DATA) cyclical = CyclicalFeatures(variables=None, drop_original=True) - X = cyclical.fit_transform(df_cyclical) - transf_df = df_cyclical.copy() - - # expected output - transf_df["day_sin"] = [ - -0.78183, - 0.0, - -0.97493, - 0.43388, - 0.78183, - 0.97493, - -0.43388, - ] - transf_df["day_cos"] = [ - 0.62349, - 1.0, - -0.222521, - -0.900969, - 0.62349, - -0.222521, - -0.900969, - ] - transf_df["months_sin"] = [ - 1.0, - -0.5, - -1.0, - 0.0, - 0.86603, - 0.0, - 0.0, - ] - transf_df["months_cos"] = [ - 0.0, - -0.86603, - -0.0, - 1.0, - -0.5, - -1.0, - 1.0, - ] - transf_df = transf_df.drop(columns=["day", "months"]) + X = cyclical.fit_transform(df) + + expected = { + "day_sin": [ + -0.78183, + 0.0, + -0.97493, + 0.43388, + 0.78183, + 0.97493, + -0.43388, + ], + "day_cos": [ + 0.62349, + 1.0, + -0.222521, + -0.900969, + 0.62349, + -0.222521, + -0.900969, + ], + "months_sin": [ + 1.0, + -0.5, + -1.0, + 0.0, + 0.86603, + 0.0, + 0.0, + ], + "months_cos": [ + 0.0, + -0.86603, + -0.0, + 1.0, + -0.5, + -1.0, + 1.0, + ], + } # test fit attr assert cyclical.max_values_ == { @@ -137,44 +142,53 @@ def test_automatically_find_variables(df_cyclical): } # test transform output - pd.testing.assert_frame_equal(X, transf_df) + assert_df_equal(X, expected) -def test_fit_raises_error_if_na_in_df(df_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): # test case 3: when dataset contains na, fit method - with pytest.raises(ValueError): - transformer = CyclicalFeatures() - transformer.fit(df_na) + df = make_df({"day": [1, 2, None, 4], "months": [1, 2, 3, 4]}) + msg = "Some of the variables in the dataset contain NaN" + with pytest.raises(ValueError, match=msg): + CyclicalFeatures().fit(df) -def test_fit_raises_error_if_user_dictionary_key_not_in_df(df_cyclical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_user_dictionary_key_not_in_df(make_df): + df = make_df(CYCLICAL_DATA) + # message differs by backend (pandas KeyError vs narwhals + # ColumnNotFoundError, a KeyError subclass), so no match= here. with pytest.raises(KeyError): - transformer = CyclicalFeatures(max_values={"dayi": 31}) - transformer.fit(df_cyclical) - + CyclicalFeatures(max_values={"dayi": 31}).fit(df) -def test_raises_error_when_init_parameters_not_permitted(df_cyclical): - with pytest.raises(TypeError): +def test_raises_error_when_init_parameters_not_permitted(): + msg = "The parameter can only take a dictionary or None" + with pytest.raises(TypeError, match=msg): # when max_values is not a dictionary CyclicalFeatures(max_values=("dayi", 31)) - with pytest.raises(ValueError): + msg = "All values in the dictionary must be integer or float" + with pytest.raises(ValueError, match=msg): # when max_values values are not integers or string CyclicalFeatures(max_values={"day": "31"}) - with pytest.raises(ValueError): + msg = "drop_original takes only boolean values True and False" + with pytest.raises(ValueError, match=msg): # when drop original is not a boolean CyclicalFeatures(drop_original="True") -def test_max_values_mapping(df_cyclical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_max_values_mapping(make_df): + df = make_df(CYCLICAL_DATA) cyclical = CyclicalFeatures(variables="day", max_values={"day": 31}) - X = cyclical.fit_transform(df_cyclical) + X = cyclical.fit_transform(df) - transf_df = df_cyclical.copy() - transf_df["day_sin"] = [ + expected = dict(CYCLICAL_DATA) + expected["day_sin"] = [ 0.937752, 0.988468, 0.848644, @@ -183,7 +197,7 @@ def test_max_values_mapping(df_cyclical): 0.394355, 0.724792, ] - transf_df["day_cos"] = [ + expected["day_cos"] = [ 0.347305, 0.151428, 0.528964, @@ -192,34 +206,43 @@ def test_max_values_mapping(df_cyclical): 0.918958, 0.688967, ] - pd.testing.assert_frame_equal(X, transf_df) + assert_df_equal(X, expected) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_features", [None, ["day", "months"], array(["day", "months"])] ) -def test_get_feature_names_out(df_cyclical, input_features): +def test_get_feature_names_out(make_df, input_features): # default features from all variables + df = make_df(CYCLICAL_DATA) transformer = CyclicalFeatures() - X = transformer.fit_transform(df_cyclical) - feat_out = list(df_cyclical.columns) + [ + X = transformer.fit_transform(df) + feat_out = list(CYCLICAL_DATA.keys()) + [ "day_sin", "day_cos", "months_sin", "months_cos", ] - assert list(X.columns) == transformer.get_feature_names_out() + assert ( + list(nw.from_native(X, eager_only=True).columns) + == transformer.get_feature_names_out() + ) assert transformer.get_feature_names_out(input_features=input_features) == feat_out - with pytest.raises(ValueError): + msg = "input_features is not equal to feature_names_in_" + with pytest.raises(ValueError, match=msg): transformer.get_feature_names_out(input_features=["day"]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=msg): transformer.get_feature_names_out(input_features=["sandia", "banana"]) transformer = CyclicalFeatures(drop_original=True) - X = transformer.fit_transform(df_cyclical) + X = transformer.fit_transform(df) feat_out = ["day_sin", "day_cos", "months_sin", "months_cos"] - assert list(X.columns) == transformer.get_feature_names_out() + assert ( + list(nw.from_native(X, eager_only=True).columns) + == transformer.get_feature_names_out() + ) assert transformer.get_feature_names_out(input_features=input_features) == feat_out diff --git a/tests/test_creation/test_decision_tree_features.py b/tests/test_creation/test_decision_tree_features.py index 4e8a93e8c..a97f68475 100644 --- a/tests/test_creation/test_decision_tree_features.py +++ b/tests/test_creation/test_decision_tree_features.py @@ -1,5 +1,9 @@ +import warnings + +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline @@ -8,44 +12,87 @@ from feature_engine.creation import DecisionTreeFeatures from tests.estimator_checks.fit_functionality_checks import check_return_empty - -@pytest.fixture(scope="module") -def df_creation(): - data = { - "Name": [ - "tom", - "nick", - "krish", - "megan", - "peter", - "jordan", - "fred", - "sam", - "alexa", - "brittany", - ], - "Age": [20, 44, 19, 33, 51, 40, 41, 37, 30, 54], - "Height": [164, 150, 178, 158, 188, 190, 168, 174, 176, 171], - "Marks": [1.0, 0.8, 0.6, 0.1, 0.3, 0.4, 0.8, 0.6, 0.5, 0.2], - } - - df = pd.DataFrame(data) - return df - - -@pytest.fixture(scope="module") -def regression_target(): - return pd.Series([4.1, 5.8, 3.9, 6.2, 4.3, 4.5, 7.2, 4.4, 4.1, 6.7]) - - -@pytest.fixture(scope="module") -def classification_target(): - return pd.Series([1, 1, 1, 0, 0, 1, 0, 1, 0, 0]) - - -@pytest.fixture(scope="module") -def multiclass_target(): - return pd.Series([1, 1, 2, 2, 0, 1, 0, 1, 0, 0]) +DATA = { + "Name": [ + "tom", + "nick", + "krish", + "megan", + "peter", + "jordan", + "fred", + "sam", + "alexa", + "brittany", + ], + "Age": [20, 44, 19, 33, 51, 40, 41, 37, 30, 54], + "Height": [164, 150, 178, 158, 188, 190, 168, 174, 176, 171], + "Marks": [1.0, 0.8, 0.6, 0.1, 0.3, 0.4, 0.8, 0.6, 0.5, 0.2], +} +REGRESSION_Y = [4.1, 5.8, 3.9, 6.2, 4.3, 4.5, 7.2, 4.4, 4.1, 6.7] +BINARY_Y = [1, 1, 1, 0, 0, 1, 0, 1, 0, 0] +MULTICLASS_Y = [1, 1, 2, 2, 0, 1, 0, 1, 0, 0] + +COMBOS = [ + "Age", + "Height", + "Marks", + ["Age", "Height"], + ["Age", "Marks"], + ["Height", "Marks"], + ["Age", "Height", "Marks"], +] + + +def _select(X, combo): + cols = combo if isinstance(combo, list) else [combo] + return nw.from_native(X, eager_only=True).select(cols).to_native() + + +def _expected_tree_predictions( + X, + y, + scoring, + random_state, + regression=True, + binary=False, + precision=None, + param_grid=None, +): + # Fits a fresh GridSearchCV per combo on the same backend as X, so this + # works as the reference for both pandas and polars input alike. + if param_grid is None: + param_grid = {"max_depth": [1, 2, 3, 4]} + if regression is True: + est = DecisionTreeRegressor(random_state=random_state) + else: + est = DecisionTreeClassifier(random_state=random_state) + tree = GridSearchCV(est, cv=3, scoring=scoring, param_grid=param_grid) + + expected = {} + for combo in COMBOS: + X_sub = _select(X, combo) + tree.fit(X_sub, y) + if regression is True: + preds = tree.predict(X_sub) + elif binary is True: + preds = tree.predict_proba(X_sub)[:, 1] + else: + preds = tree.predict(X_sub) + if precision is not None: + preds = np.round(preds, precision) + expected[f"tree({combo})"] = list(preds) + return expected + + +def assert_df_equal(X, expected: dict) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + if all(isinstance(v, (int, float, np.integer, np.floating)) for v in values): + assert result[col] == pytest.approx(values, abs=1e-6) + else: + assert result[col] == values @pytest.mark.parametrize("precision", ["string", 0.1, -1, np.nan]) @@ -204,390 +251,226 @@ def test_create_variable_combinations_when_tuple(input_features, expected): assert combos == expected -def test_feature_creation_regression(df_creation, regression_target): - X = df_creation.copy() - y = regression_target.copy() - +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_regression(make_df): + X = make_df(DATA) scoring = "neg_mean_squared_error" rs = 0 tr = DecisionTreeFeatures(scoring=scoring, random_state=rs) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeRegressor(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - X_exp[varn] = tree.predict(X[combon].to_frame()) - else: - tree.fit(X[combon], y) - X_exp[varn] = tree.predict(X[combon]) - - pd.testing.assert_frame_equal(Xt, X_exp) + Xt = tr.fit_transform(X, REGRESSION_Y) + expected = dict(DATA) + expected.update(_expected_tree_predictions(X, REGRESSION_Y, scoring, rs)) + assert_df_equal(Xt, expected) -def test_feature_creation_regression_and_precision(df_creation, regression_target): - X = df_creation.copy() - y = regression_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_regression_and_precision(make_df): + X = make_df(DATA) scoring = "neg_mean_squared_error" rs = 0 tr = DecisionTreeFeatures(scoring=scoring, random_state=rs, precision=1) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeRegressor(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - preds = tree.predict(X[combon].to_frame()) - X_exp[varn] = np.round(preds, 1) - else: - tree.fit(X[combon], y) - preds = tree.predict(X[combon]) - X_exp[varn] = np.round(preds, 1) - - pd.testing.assert_frame_equal(Xt, X_exp) + Xt = tr.fit_transform(X, REGRESSION_Y) + expected = dict(DATA) + expected.update( + _expected_tree_predictions(X, REGRESSION_Y, scoring, rs, precision=1) + ) + assert_df_equal(Xt, expected) -def test_feature_creation_regression_drop_original(df_creation, regression_target): - X = df_creation.copy() - y = regression_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_regression_drop_original(make_df): + X = make_df(DATA) scoring = "neg_mean_squared_error" rs = 0 tr = DecisionTreeFeatures(scoring=scoring, random_state=rs, drop_original=True) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeRegressor(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - X_exp[varn] = tree.predict(X[combon].to_frame()) - else: - tree.fit(X[combon], y) - X_exp[varn] = tree.predict(X[combon]) - X_exp.drop(["Age", "Height", "Marks"], axis=1, inplace=True) - - pd.testing.assert_frame_equal(Xt, X_exp) + Xt = tr.fit_transform(X, REGRESSION_Y) + expected = {"Name": DATA["Name"]} + expected.update(_expected_tree_predictions(X, REGRESSION_Y, scoring, rs)) + assert_df_equal(Xt, expected) -def test_feature_creation_binary_classif(df_creation, classification_target): - X = df_creation.copy() - y = classification_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_binary_classif(make_df): + X = make_df(DATA) scoring = "roc_auc" rs = 0 tr = DecisionTreeFeatures(scoring=scoring, random_state=rs, regression=False) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeClassifier(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - preds = tree.predict_proba(X[combon].to_frame()) - X_exp[varn] = preds[:, 1] - else: - tree.fit(X[combon], y) - preds = tree.predict_proba(X[combon]) - X_exp[varn] = preds[:, 1] - - pd.testing.assert_frame_equal(Xt, X_exp) + Xt = tr.fit_transform(X, BINARY_Y) + expected = dict(DATA) + expected.update( + _expected_tree_predictions( + X, BINARY_Y, scoring, rs, regression=False, binary=True + ) + ) + assert_df_equal(Xt, expected) -def test_feature_creation_binary_classif_w_precision( - df_creation, classification_target -): - X = df_creation.copy() - y = classification_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_binary_classif_w_precision(make_df): + X = make_df(DATA) scoring = "roc_auc" rs = 0 tr = DecisionTreeFeatures( scoring=scoring, random_state=rs, regression=False, precision=2 ) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeClassifier(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - preds = tree.predict_proba(X[combon].to_frame()) - X_exp[varn] = np.round(preds[:, 1], 2) - else: - tree.fit(X[combon], y) - preds = tree.predict_proba(X[combon]) - X_exp[varn] = np.round(preds[:, 1], 2) - - pd.testing.assert_frame_equal(Xt, X_exp) + Xt = tr.fit_transform(X, BINARY_Y) + expected = dict(DATA) + expected.update( + _expected_tree_predictions( + X, BINARY_Y, scoring, rs, regression=False, binary=True, precision=2 + ) + ) + assert_df_equal(Xt, expected) -def test_feature_creation_binary_multiclass(df_creation, multiclass_target): - X = df_creation.copy() - y = multiclass_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_feature_creation_binary_multiclass(make_df): + X = make_df(DATA) scoring = "roc_auc" rs = 0 tr = DecisionTreeFeatures(scoring=scoring, random_state=rs, regression=False) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeClassifier(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - preds = tree.predict(X[combon].to_frame()) - X_exp[varn] = preds - else: - tree.fit(X[combon], y) - preds = tree.predict(X[combon]) - X_exp[varn] = preds - - pd.testing.assert_frame_equal(Xt, X_exp) - - -def test_get_feature_names_out(df_creation, regression_target): - X = df_creation.copy() - y = regression_target.copy() + Xt = tr.fit_transform(X, MULTICLASS_Y) - tr = DecisionTreeFeatures( - variables=["Age", "Marks"], + expected = dict(DATA) + expected.update( + _expected_tree_predictions( + X, MULTICLASS_Y, scoring, rs, regression=False, binary=False + ) ) + assert_df_equal(Xt, expected) - Xt = tr.fit_transform(X, y) - feat_out = Xt.columns.to_list() - assert tr.get_feature_names_out() == feat_out - assert tr.get_feature_names_out(X.columns.to_list()) == feat_out +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out(make_df): + X = make_df(DATA) + tr = DecisionTreeFeatures(variables=["Age", "Marks"]) + Xt = tr.fit_transform(X, REGRESSION_Y) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) + assert tr.get_feature_names_out() == feat_out + assert tr.get_feature_names_out(list(DATA.keys())) == feat_out -def test_get_feature_names_out_from_pipeline(df_creation, regression_target): - X = df_creation.copy() - y = regression_target.copy() - - # set up transformer - tr = DecisionTreeFeatures( - variables=["Age", "Marks"], - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_from_pipeline(make_df): + X = make_df(DATA) + tr = DecisionTreeFeatures(variables=["Age", "Marks"]) pipe = Pipeline([("transformer", tr)]) - - Xt = pipe.fit_transform(X, y) - feat_out = Xt.columns.to_list() + Xt = pipe.fit_transform(X, REGRESSION_Y) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) assert pipe.get_feature_names_out(input_features=None) == feat_out - assert pipe.get_feature_names_out(input_features=X.columns.to_list()) == feat_out + assert pipe.get_feature_names_out(input_features=list(DATA.keys())) == feat_out +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) -def test_get_feature_names_out_raises_error_when_wrong_param( - _input_features, df_creation, regression_target -): - X = df_creation.copy() - y = regression_target.copy() - - tr = DecisionTreeFeatures( - variables=["Age", "Marks"], - ) - tr.fit(X, y) - +def test_get_feature_names_out_raises_error_when_wrong_param(make_df, _input_features): + X = make_df(DATA) + tr = DecisionTreeFeatures(variables=["Age", "Marks"]) + tr.fit(X, REGRESSION_Y) with pytest.raises(ValueError): tr.get_feature_names_out(input_features=_input_features) -def test_error_when_regression_true_and_target_binary( - df_creation, classification_target -): - X = df_creation.copy() - y = classification_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_regression_true_and_target_binary(make_df): + X = make_df(DATA) tr = DecisionTreeFeatures(regression=True) msg = ( "Trying to fit a regression to a binary target is not " - + "allowed by this transformer. Check the target values " - + "or set regression to False." + "allowed by this transformer. Check the target values " + "or set regression to False." ) with pytest.raises(ValueError, match=msg): - tr.fit(X, y) + tr.fit(X, BINARY_Y) -def test_user_enter_param_grid(df_creation, classification_target): - X = df_creation.copy() - y = classification_target.copy() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_user_enter_param_grid(make_df): + X = make_df(DATA) scoring = "roc_auc" rs = 0 grid = {"max_depth": [1, 2, 3, 4]} tr = DecisionTreeFeatures( scoring=scoring, random_state=rs, regression=False, param_grid=grid ) - Xt = tr.fit_transform(X, y) - - # get expected - est = DecisionTreeClassifier(random_state=rs) - tree = GridSearchCV( - est, - cv=3, - scoring=scoring, - param_grid={"max_depth": [1, 2, 3, 4]}, - ) - - combos = [ - "Age", - "Height", - "Marks", - ["Age", "Height"], - ["Age", "Marks"], - ["Height", "Marks"], - ["Age", "Height", "Marks"], - ] - var_names = [f"tree({item})" for item in combos] - - X_exp = df_creation.copy() - for i in range(len(combos)): - varn = var_names[i] - combon = combos[i] - if isinstance(combon, str): - tree.fit(X[combon].to_frame(), y) - preds = tree.predict_proba(X[combon].to_frame()) - X_exp[varn] = preds[:, 1] - else: - tree.fit(X[combon], y) - preds = tree.predict_proba(X[combon]) - X_exp[varn] = preds[:, 1] + Xt = tr.fit_transform(X, BINARY_Y) - pd.testing.assert_frame_equal(Xt, X_exp) + expected = dict(DATA) + expected.update( + _expected_tree_predictions( + X, BINARY_Y, scoring, rs, regression=False, binary=True, param_grid=grid + ) + ) + assert_df_equal(Xt, expected) def test_check_return_empty(): # DecisionTreeFeatures is not part of the check_feature_engine_estimator # pipeline (test_check_estimator_creation.py only feeds MathFeatures, # RelativeFeatures and CyclicalFeatures into it), so return_empty is - # tested directly here instead. + # tested directly here instead. check_return_empty is a shared, + # pandas-only estimator-check helper used across the library. check_return_empty(DecisionTreeFeatures(regression=False)) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_n_jobs_parallel_matches_sequential(make_df): + # core correctness check for n_jobs: parallelizing tree training across + # feature combinations must produce identical trees, and therefore + # identical predictions, to sequential training (n_jobs=None). + X = make_df(DATA) + tr_seq = DecisionTreeFeatures(n_jobs=None, random_state=0) + tr_seq.fit(X, REGRESSION_Y) + tr_par = DecisionTreeFeatures(n_jobs=2, random_state=0) + tr_par.fit(X, REGRESSION_Y) + + Xt_seq = tr_seq.transform(X) + Xt_par = tr_par.transform(X) + + expected = nw.from_native(Xt_seq, eager_only=True).to_dict(as_series=False) + assert_df_equal(Xt_par, expected) + + +def test_transform_does_not_fragment_pandas_output(): + # regression test: transform() used to assign one new tree column at a + # time (X[col_name] = preds), which triggers pandas' "DataFrame is + # highly fragmented" PerformanceWarning once there are enough feature + # combinations - fixed by building all new columns in one DataFrame + # and joining once. Needs enough variables to cross pandas' internal + # fragmentation threshold (a handful of combos won't trigger it). + rng = np.random.RandomState(0) + n_vars = 9 + X = pd.DataFrame( + rng.rand(200, n_vars), columns=[f"v{i}" for i in range(n_vars)] + ) + y = rng.rand(200) + + tr = DecisionTreeFeatures( + features_to_combine=3, param_grid={"max_depth": [1, 2]}, random_state=0 + ) + tr.fit(X, y) + + with warnings.catch_warnings(): + warnings.simplefilter("error", pd.errors.PerformanceWarning) + tr.transform(X) + + +def test_single_int_named_feature_combo(): + # regression test: a single-variable combo with an integer column name + # used to crash (isinstance(features, str) missed the int case), since + # X[features] for a bare int returns a 1D Series, not the 2D input + # sklearn requires - fixed to check isinstance(features, (str, int)). + # Integer column names are pandas-only - polars requires string columns. + df = pd.DataFrame({0: [1.0, 2, 3, 4, 5, 6, 7, 8], 1: [2.0, 3, 4, 5, 6, 7, 8, 9]}) + y = [1.0, 2, 3, 4, 5, 6, 7, 8] + transformer = DecisionTreeFeatures(features_to_combine=1, random_state=0) + transformer.fit(df, y) + Xt = transformer.transform(df) + assert "tree(0)" in Xt.columns + assert "tree(1)" in Xt.columns diff --git a/tests/test_creation/test_geo_features.py b/tests/test_creation/test_geo_features.py index bbd800044..f137e4ef1 100644 --- a/tests/test_creation/test_geo_features.py +++ b/tests/test_creation/test_geo_features.py @@ -1,81 +1,84 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.creation import GeoDistanceFeatures +COORDS_DATA = { + "lat1": [40.7128], + "lon1": [-74.0060], + "lat2": [34.0522], + "lon2": [-118.2437], +} -@pytest.fixture -def df_coords(): - """Fixture providing sample coordinate data for a single route.""" - return pd.DataFrame({ - "lat1": [40.7128], - "lon1": [-74.0060], - "lat2": [34.0522], - "lon2": [-118.2437], - }) - - -@pytest.fixture -def df_multi_coords(): - """Fixture providing sample coordinate data with multiple rows.""" - return pd.DataFrame({ - "origin_lat": [40.7128, 34.0522, 41.8781], - "origin_lon": [-74.0060, -118.2437, -87.6298], - "dest_lat": [34.0522, 41.8781, 40.7128], - "dest_lon": [-118.2437, -87.6298, -74.0060], - }) - - -@pytest.fixture -def df_with_extra(): - """Fixture for DataFrame with coordinates and extra columns.""" - return pd.DataFrame({ - "lat1": [40.0], - "lon1": [-74.0], - "lat2": [34.0], - "lon2": [-118.0], - "other": [1], - }) - - -def test_haversine_distance_default(df_coords): +MULTI_COORDS_DATA = { + "origin_lat": [40.7128, 34.0522, 41.8781], + "origin_lon": [-74.0060, -118.2437, -87.6298], + "dest_lat": [34.0522, 41.8781, 40.7128], + "dest_lon": [-118.2437, -87.6298, -74.0060], +} + +COORDS_WITH_EXTRA_DATA = { + "lat1": [40.0], + "lon1": [-74.0], + "lat2": [34.0], + "lon2": [-118.0], + "other": [1], +} + + +def get_value(X, col: str, idx: int = 0): + """Extract a single scalar from a pandas or polars dataframe column.""" + return nw.from_native(X, eager_only=True).get_column(col).to_list()[idx] + + +def assert_df_equal(X, expected: dict, abs_tol: float = 1e-5) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert result[col] == pytest.approx(values, abs=abs_tol) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_haversine_distance_default(make_df): """Test Haversine distance calculation with default parameters.""" + df = make_df(COORDS_DATA) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) - X_tr = transformer.fit_transform(df_coords) + X_tr = transformer.fit_transform(df) assert "geo_distance" in X_tr.columns - assert 3900 < X_tr["geo_distance"].iloc[0] < 4000 + assert 3900 < get_value(X_tr, "geo_distance") < 4000 -def test_haversine_distance_miles(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_haversine_distance_miles(make_df): """Test Haversine distance in miles.""" - X = pd.DataFrame({ - "lat1": [40.7128], - "lon1": [-74.0060], - "lat2": [34.0522], - "lon2": [-118.2437], - }) + X = make_df(COORDS_DATA) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", output_unit="miles" ) X_tr = transformer.fit_transform(X) - assert 2400 < X_tr["geo_distance"].iloc[0] < 2500 + assert 2400 < get_value(X_tr, "geo_distance") < 2500 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("method", ["haversine", "euclidean", "manhattan"]) @pytest.mark.parametrize("output_unit", ["km", "miles", "meters", "feet"]) -def test_same_location_zero_distance(method, output_unit): +def test_same_location_zero_distance(make_df, method, output_unit): """Test that same location returns zero distance for all methods and units.""" - X = pd.DataFrame({ - "lat1": [40.7128, 34.0522], - "lon1": [-74.0060, -118.2437], - "lat2": [40.7128, 34.0522], - "lon2": [-74.0060, -118.2437], - }) + X = make_df( + { + "lat1": [40.7128, 34.0522], + "lon1": [-74.0060, -118.2437], + "lat2": [40.7128, 34.0522], + "lon2": [-74.0060, -118.2437], + } + ) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", @@ -86,14 +89,14 @@ def test_same_location_zero_distance(method, output_unit): ) X_tr = transformer.fit_transform(X) - np.testing.assert_array_almost_equal( - X_tr["geo_distance"].values, [0.0, 0.0], decimal=10 - ) + values = nw.from_native(X_tr, eager_only=True).get_column("geo_distance") + np.testing.assert_array_almost_equal(values.to_list(), [0.0, 0.0], decimal=10) -def test_euclidean_method(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_euclidean_method(make_df): """Test Euclidean distance method returns expected values.""" - X = pd.DataFrame({"lat1": [0.0], "lon1": [0.0], "lat2": [1.0], "lon2": [1.0]}) + X = make_df({"lat1": [0.0], "lon1": [0.0], "lat2": [1.0], "lon2": [1.0]}) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", method="euclidean" ) @@ -101,13 +104,14 @@ def test_euclidean_method(): expected_distance = np.sqrt(2) * 111.0 np.testing.assert_almost_equal( - X_tr["geo_distance"].iloc[0], expected_distance, decimal=1 + get_value(X_tr, "geo_distance"), expected_distance, decimal=1 ) -def test_manhattan_method(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_manhattan_method(make_df): """Test Manhattan distance method returns expected values.""" - X = pd.DataFrame({"lat1": [0.0], "lon1": [0.0], "lat2": [1.0], "lon2": [1.0]}) + X = make_df({"lat1": [0.0], "lon1": [0.0], "lat2": [1.0], "lon2": [1.0]}) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", method="manhattan" ) @@ -115,30 +119,35 @@ def test_manhattan_method(): expected_distance = 2 * 111.0 np.testing.assert_almost_equal( - X_tr["geo_distance"].iloc[0], expected_distance, decimal=1 + get_value(X_tr, "geo_distance"), expected_distance, decimal=1 ) -def test_custom_output_column_name(df_coords): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_custom_output_column_name(make_df): """Test custom output column name.""" + df = make_df(COORDS_DATA) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", output_col="distance_km" ) - X_tr = transformer.fit_transform(df_coords) + X_tr = transformer.fit_transform(df) assert "distance_km" in X_tr.columns assert "geo_distance" not in X_tr.columns -def test_drop_original_columns(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_drop_original_columns(make_df): """Test drop_original parameter removes coordinate columns.""" - X = pd.DataFrame({ - "lat1": [40.7128], - "lon1": [-74.0060], - "lat2": [34.0522], - "lon2": [-118.2437], - "other": [1], - }) + X = make_df( + { + "lat1": [40.7128], + "lon1": [-74.0060], + "lat2": [34.0522], + "lon2": [-118.2437], + "other": [1], + } + ) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", drop_original=True ) @@ -153,26 +162,23 @@ def test_drop_original_columns(): assert list(X_tr.columns) == ["other", "geo_distance"] -def test_multiple_rows(df_multi_coords): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_multiple_rows(make_df): """Test transformation with multiple rows returns expected distances.""" + df = make_df(MULTI_COORDS_DATA) transformer = GeoDistanceFeatures( lat1="origin_lat", lon1="origin_lon", lat2="dest_lat", lon2="dest_lon" ) - X_tr = transformer.fit_transform(df_multi_coords) + X_tr = transformer.fit_transform(df) - expected = df_multi_coords.copy() + expected = dict(MULTI_COORDS_DATA) expected["geo_distance"] = [ 3935.746254609723, 2803.971506975193, 1144.2912739463475, ] - pd.testing.assert_frame_equal( - X_tr, - expected, - check_exact=False, - atol=0.001, - ) + assert_df_equal(X_tr, expected, abs_tol=0.001) @pytest.mark.parametrize("invalid_method", ["invalid", True, 123]) @@ -197,9 +203,10 @@ def test_invalid_output_unit_raises_error(invalid_unit): ) -def test_missing_columns_raises_error(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_missing_columns_raises_error(make_df): """Test that missing columns raise ValueError on fit.""" - X = pd.DataFrame({"lat1": [1], "lon1": [1]}) + X = make_df({"lat1": [1], "lon1": [1]}) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) @@ -207,15 +214,18 @@ def test_missing_columns_raises_error(): transformer.fit(X) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("invalid_lat", [100, -100]) -def test_invalid_latitude_range_raises_error(invalid_lat): +def test_invalid_latitude_range_raises_error(make_df, invalid_lat): """Test that latitude outside [-90, 90] raises ValueError.""" - X = pd.DataFrame({ - "lat1": [invalid_lat], - "lon1": [0], - "lat2": [0], - "lon2": [0], - }) + X = make_df( + { + "lat1": [invalid_lat], + "lon1": [0], + "lat2": [0], + "lon2": [0], + } + ) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) @@ -223,15 +233,18 @@ def test_invalid_latitude_range_raises_error(invalid_lat): transformer.fit(X) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("invalid_lon", [200, -200]) -def test_invalid_longitude_range_raises_error(invalid_lon): +def test_invalid_longitude_range_raises_error(make_df, invalid_lon): """Test that longitude outside [-180, 180] raises ValueError.""" - X = pd.DataFrame({ - "lat1": [0], - "lon1": [invalid_lon], - "lat2": [0], - "lon2": [0], - }) + X = make_df( + { + "lat1": [0], + "lon1": [invalid_lon], + "lat2": [0], + "lon2": [0], + } + ) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) @@ -239,14 +252,17 @@ def test_invalid_longitude_range_raises_error(invalid_lon): transformer.fit(X) -def test_validate_ranges_disabled(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_validate_ranges_disabled(make_df): """Test that invalid coordinates don't raise error when validate_ranges=False.""" - X = pd.DataFrame({ - "lat1": [100], - "lon1": [200], - "lat2": [0], - "lon2": [0], - }) + X = make_df( + { + "lat1": [100], + "lon1": [200], + "lat2": [0], + "lon2": [0], + } + ) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", validate_ranges=False ) @@ -268,11 +284,10 @@ def test_validate_ranges_parameter_validation(invalid_value): ) -def test_fit_stores_attributes(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_stores_attributes(make_df): """Test that fit stores expected attributes with correct values.""" - X = pd.DataFrame( - {"lat1": [40.0], "lon1": [-74.0], "lat2": [34.0], "lon2": [-118.0]} - ) + X = make_df({"lat1": [40.0], "lon1": [-74.0], "lat2": [34.0], "lon2": [-118.0]}) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) @@ -286,38 +301,38 @@ def test_fit_stores_attributes(): assert transformer.n_features_in_ == 4 -def test_get_feature_names_out(df_with_extra): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out(make_df): """Test get_feature_names_out returns correct feature names.""" + df = make_df(COORDS_WITH_EXTRA_DATA) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2" ) - transformer.fit(df_with_extra) + transformer.fit(df) feature_names = transformer.get_feature_names_out() expected_names = ["lat1", "lon1", "lat2", "lon2", "other", "geo_distance"] assert feature_names == expected_names -def test_get_feature_names_out_with_drop_original(df_with_extra): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_with_drop_original(make_df): """Test get_feature_names_out when drop_original=True.""" + df = make_df(COORDS_WITH_EXTRA_DATA) transformer = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", drop_original=True ) - transformer.fit(df_with_extra) + transformer.fit(df) feature_names = transformer.get_feature_names_out() expected_names = ["other", "geo_distance"] assert feature_names == expected_names -def test_output_units_conversion(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_output_units_conversion(make_df): """Test different output units give consistent results with correct conversion.""" - X = pd.DataFrame({ - "lat1": [40.7128], - "lon1": [-74.0060], - "lat2": [34.0522], - "lon2": [-118.2437], - }) + data = COORDS_DATA transformer_km = GeoDistanceFeatures( lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", output_unit="km" @@ -326,8 +341,10 @@ def test_output_units_conversion(): lat1="lat1", lon1="lon1", lat2="lat2", lon2="lon2", output_unit="miles" ) - dist_km = transformer_km.fit_transform(X.copy())["geo_distance"].iloc[0] - dist_miles = transformer_miles.fit_transform(X.copy())["geo_distance"].iloc[0] + dist_km = get_value(transformer_km.fit_transform(make_df(data)), "geo_distance") + dist_miles = get_value( + transformer_miles.fit_transform(make_df(data)), "geo_distance" + ) expected_miles = dist_km * 0.621371 np.testing.assert_almost_equal(dist_miles, expected_miles, decimal=0) @@ -356,7 +373,5 @@ def test_more_tags_and_sklearn_tags(): == "transformer has mandatory parameters" ) - # basic check for sklearn tags if available (new sklearn versions) - if hasattr(transformer, "__sklearn_tags__"): - tags = transformer.__sklearn_tags__() - assert tags is not None + tags = transformer.__sklearn_tags__() + assert tags is not None diff --git a/tests/test_creation/test_math_features.py b/tests/test_creation/test_math_features.py index 9c4c6b10c..1c695ca88 100644 --- a/tests/test_creation/test_math_features.py +++ b/tests/test_creation/test_math_features.py @@ -1,13 +1,35 @@ import warnings +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.pipeline import Pipeline from feature_engine.creation import MathFeatures -dob_datrange = pd.date_range("2020-02-24", periods=4, freq="min") +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + + +def _none_to_nan(values): + # Missing values print as None for polars, NaN for pandas float columns + # - both mean "missing" here, so normalize both sides before comparing. + return [np.nan if v is None else v for v in values] + + +def assert_df_equal(X, expected: dict, abs_tol: float = 1e-5) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert _none_to_nan(result[col]) == pytest.approx( + _none_to_nan(values), abs=abs_tol, nan_ok=True + ) # test param variables_to_combine @@ -83,138 +105,95 @@ def test_error_new_variable_names_not_permitted(): ) -def test_aggregations_with_strings(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_aggregations_with_strings(make_df): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "prod", "mean", "std", "max", "min"] ) - X = transformer.fit_transform(df_vartypes) + Xt = transformer.fit_transform(df) - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], - "prod_Age_Marks": [18.0, 16.8, 13.299999999999999, 10.799999999999999], - "mean_Age_Marks": [10.45, 10.9, 9.85, 9.3], - "std_Age_Marks": [ - 13.505739520663058, - 14.28355697996826, - 12.94005409571382, - 12.303657992645928, - ], - "max_Age_Marks": [20.0, 21.0, 19.0, 18.0], - "min_Age_Marks": [0.9, 0.8, 0.7, 0.6], - } - ) + expected = dict(DATA) + expected["sum_Age_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["prod_Age_Marks"] = [18.0, 16.8, 13.3, 10.8] + expected["mean_Age_Marks"] = [10.45, 10.9, 9.85, 9.3] + expected["std_Age_Marks"] = [13.505740, 14.283557, 12.940054, 12.303658] + expected["max_Age_Marks"] = [20.0, 21.0, 19.0, 18.0] + expected["min_Age_Marks"] = [0.9, 0.8, 0.7, 0.6] - # transform params - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) -def test_aggregations_with_functions(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_aggregations_with_functions(make_df): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=[np.sum, np.mean, np.std] ) - X = transformer.fit_transform(df_vartypes) + Xt = transformer.fit_transform(df) - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], - "mean_Age_Marks": [10.45, 10.9, 9.85, 9.3], - "std_Age_Marks": [ - 13.505739520663058, - 14.28355697996826, - 12.94005409571382, - 12.303657992645928, - ], - } - ) + expected = dict(DATA) + expected["sum_Age_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["mean_Age_Marks"] = [10.45, 10.9, 9.85, 9.3] - # TODO: Remove pandas < 3 support when dropping older pandas versions - # In pandas >=3, when the user passes np.std, agg will use numpy. - # In pandas <3, when the user passes np.std, agg will use pd.std. - # Hence the difference in results - if pd.__version__ >= "3": - ref["std_Age_Marks"] = np.std(df_vartypes[["Age", "Marks"]], axis=1) + # np.std uses ddof=0 (population std) everywhere now, except pandas < 3, + # where agg() still routes np.std through pandas' own ddof=1 Series.std(). + # TODO: remove the pandas < 3 branch when dropping older pandas support. + if make_df is pd.DataFrame and int(pd.__version__.split(".")[0]) < 3: + expected["std_Age_Marks"] = [13.505740, 14.283557, 12.940054, 12.303658] + else: + arr = np.array([DATA["Age"], DATA["Marks"]], dtype=float) + expected["std_Age_Marks"] = np.std(arr, axis=0).tolist() - # transform params - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) -def test_user_enters_two_operations(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_user_enters_two_operations(make_df): + df = make_df(DATA) transformer = MathFeatures(variables=["Age", "Marks"], func=["sum", np.mean]) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["sum_Age_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["mean_Age_Marks"] = [10.45, 10.9, 9.85, 9.3] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], - "mean_Age_Marks": [10.45, 10.9, 9.85, 9.3], - } - ) - - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) -def test_new_variable_names(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_new_variable_names(make_df): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], new_variables_names=["sum_of_two_vars", "mean_of_two_vars"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["sum_of_two_vars"] = [20.9, 21.8, 19.7, 18.6] + expected["mean_of_two_vars"] = [10.45, 10.9, 9.85, 9.3] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_of_two_vars": [20.9, 21.8, 19.7, 18.6], - "mean_of_two_vars": [10.45, 10.9, 9.85, 9.3], - } - ) + assert_df_equal(Xt, expected) - pd.testing.assert_frame_equal(X, ref) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_one_mathematical_operation(make_df): + df = make_df(DATA) + expected = dict(DATA) + expected["sum_Age_Marks"] = [20.9, 21.8, 19.7, 18.6] -def test_one_mathematical_operation(df_vartypes): transformer = MathFeatures(variables=["Age", "Marks"], func="sum") - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], - } - ) - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(transformer.fit_transform(df), expected) transformer = MathFeatures(variables=["Age", "Marks"], func=["sum"]) - X = transformer.fit_transform(df_vartypes) - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(transformer.fit_transform(df), expected) def test_variable_names_when_df_cols_are_integers(df_numeric_columns): + # polars requires string column names, so int-named columns are + # pandas-only - no polars equivalent to parametrize against here. transformer = MathFeatures( variables=[2, 3], func=["sum", "prod", "mean", "std", "max", "min"] ) @@ -227,7 +206,7 @@ def test_variable_names_when_df_cols_are_integers(df_numeric_columns): 1: ["London", "Manchester", "Liverpool", "Bristol"], 2: [20, 21, 19, 18], 3: [0.9, 0.8, 0.7, 0.6], - 4: dob_datrange, + 4: pd.date_range("2020-02-24", periods=4, freq="min"), "sum_2_3": [20.9, 21.8, 19.7, 18.6], "prod_2_3": [18.0, 16.8, 13.299999999999999, 10.799999999999999], "mean_2_3": [10.45, 10.9, 9.85, 9.3], @@ -245,9 +224,11 @@ def test_variable_names_when_df_cols_are_integers(df_numeric_columns): pd.testing.assert_frame_equal(X, ref) -def test_error_when_null_values_in_variable(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_null_values_in_variable(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) math_combinator = MathFeatures( variables=["Age", "Marks"], @@ -258,65 +239,65 @@ def test_error_when_null_values_in_variable(df_vartypes): with pytest.raises(ValueError): math_combinator.fit(df_na) - math_combinator.fit(df_vartypes) + math_combinator.fit(make_df(DATA)) with pytest.raises(ValueError): math_combinator.transform(df_na) -def test_no_error_when_null_values_in_variable(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_no_error_when_null_values_in_variable(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], missing_values="ignore", ) + Xt = transformer.fit_transform(df_na) - X = transformer.fit_transform(df_na) + expected = dict(data_na) + expected["sum_Age_Marks"] = [20.9, 0.8, 19.7, 18.6] + expected["mean_Age_Marks"] = [10.45, 0.8, 9.85, 9.3] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, np.nan, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 0.8, 19.7, 18.6], - "mean_Age_Marks": [10.45, 0.8, 9.85, 9.3], - } - ) - # transform params - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) -def test_standard_aggregations_match_pandas_with_missing_values(): - X = pd.DataFrame( - { - "a": [1.0, np.nan, np.nan, 4.0], - "b": [3.0, 4.0, np.nan, 6.0], - "c": [5.0, 8.0, np.nan, np.nan], - } - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_standard_aggregations_match_pandas_with_missing_values(make_df): + data = { + "a": [1.0, np.nan, np.nan, 4.0], + "b": [3.0, 4.0, np.nan, 6.0], + "c": [5.0, 8.0, np.nan, np.nan], + } functions = ["sum", "mean", "std", "var", "min", "max", "prod", "median"] names = [f"result_{function}" for function in functions] + + # pandas' own agg() is the ground truth both backends are checked against. + X_pd = pd.DataFrame(data) with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) - expected = X.agg(functions, axis=1) - expected.columns = names + expected_df = X_pd.agg(functions, axis=1) + expected = {name: expected_df[fn].tolist() for name, fn in zip(names, functions)} + df = make_df(data) transformer = MathFeatures( - variables=list(X.columns), + variables=list(data.keys()), func=functions, new_variables_names=names, missing_values="ignore", ) - result = transformer.fit_transform(X) + result = transformer.fit_transform(df) - pd.testing.assert_frame_equal(result[names], expected) + result_dict = nw.from_native(result, eager_only=True).to_dict(as_series=False) + for name in names: + assert result_dict[name] == pytest.approx(expected[name], nan_ok=True) def test_nullable_dtypes_use_backwards_compatible_aggregation(): + # pandas' nullable "Int64" dtype is pandas-specific - no polars + # equivalent to parametrize against here. X = pd.DataFrame( { "a": pd.Series([1, pd.NA, 3], dtype="Int64"), @@ -339,65 +320,111 @@ def test_nullable_dtypes_use_backwards_compatible_aggregation(): pd.testing.assert_frame_equal(result[names], expected) -def test_custom_function_uses_pandas_aggregation_fallback(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_custom_function_fallback(make_df): + # max()/min()/sum() are built-ins, so they work identically whether + # func receives a pandas Series (pandas' agg(axis=1) fallback) or a + # plain tuple (polars' map_rows fallback) - one callable, one test. def peak_to_peak(row): - return row.max() - row.min() + return max(row) - min(row) - expected = df_vartypes[["Age", "Marks"]].agg(peak_to_peak, axis=1) + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=peak_to_peak, new_variables_names=["age_marks_range"], ) + Xt = transformer.fit_transform(df) - result = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["age_marks_range"] = [ + a - m for a, m in zip(DATA["Age"], DATA["Marks"]) + ] + assert_df_equal(Xt, expected) - pd.testing.assert_series_equal( - result["age_marks_range"], expected, check_names=False - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_multiple_custom_functions_fallback(make_df): + def total(row): + return sum(row) -def test_drop_original_variables(df_vartypes): + def spread(row): + return max(row) - min(row) + + df = make_df(DATA) + transformer = MathFeatures( + variables=["Age", "Marks"], + func=[total, spread], + new_variables_names=["total", "spread"], + ) + Xt = transformer.fit_transform(df) + + expected = dict(DATA) + expected["total"] = [a + m for a, m in zip(DATA["Age"], DATA["Marks"])] + expected["spread"] = [a - m for a, m in zip(DATA["Age"], DATA["Marks"])] + assert_df_equal(Xt, expected) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_uncommon_aggregation_string_only_supported_for_pandas(make_df): + # a genuine, documented backend asymmetry, not an oversight: pandas' + # agg() accepts any of its own aggregation strings (even ones outside + # our NumPy-vectorized table), but polars has no way to resolve an + # arbitrary pandas-specific string without pandas itself, so it raises + # instead of silently doing the wrong thing. + df = make_df(DATA) + transformer = MathFeatures(variables=["Age", "Marks"], func="sem") + + if make_df is pd.DataFrame: + Xt = transformer.fit_transform(df) + expected = dict(DATA) + expected["sem_Age_Marks"] = [9.55, 10.10, 9.15, 8.70] + assert_df_equal(Xt, expected) + else: + with pytest.raises(NotImplementedError, match="has no NumPy-vectorized"): + transformer.fit_transform(df) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_drop_original_variables(make_df): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], drop_original=True, ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "dob": dob_datrange, - "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], - "mean_Age_Marks": [10.45, 10.9, 9.85, 9.3], - } - ) - - pd.testing.assert_frame_equal(X, ref) + expected = { + "Name": DATA["Name"], + "City": DATA["City"], + "sum_Age_Marks": [20.9, 21.8, 19.7, 18.6], + "mean_Age_Marks": [10.45, 10.9, 9.85, 9.3], + } + assert_df_equal(Xt, expected) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_varnames", [None, ["var1", "var2"]]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out(_varnames, _drop, df_vartypes): +def test_get_feature_names_out(make_df, _varnames, _drop): + df = make_df(DATA) tr = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], new_variables_names=_varnames, drop_original=_drop, ) - X = tr.fit_transform(df_vartypes) - feat_out = list(X.columns) + Xt = tr.fit_transform(df) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) assert tr.get_feature_names_out(input_features=None) == feat_out - assert tr.get_feature_names_out(input_features=df_vartypes.columns) == feat_out +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_varnames", [None, ["var1", "var2"]]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out_from_pipeline(_varnames, _drop, df_vartypes): - # set up transformer +def test_get_feature_names_out_from_pipeline(make_df, _varnames, _drop): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], @@ -406,24 +433,21 @@ def test_get_feature_names_out_from_pipeline(_varnames, _drop, df_vartypes): ) pipe = Pipeline([("transformer", transformer)]) + Xt = pipe.fit_transform(df) - # fit transformer - X = pipe.fit_transform(df_vartypes) - - feat_out = list(X.columns) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) assert pipe.get_feature_names_out(input_features=None) == feat_out - assert pipe.get_feature_names_out(input_features=df_vartypes.columns) == feat_out +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) -def test_get_feature_names_out_raises_error_when_wrong_param( - _input_features, df_vartypes -): +def test_get_feature_names_out_raises_error_when_wrong_param(make_df, _input_features): + df = make_df(DATA) transformer = MathFeatures( variables=["Age", "Marks"], func=["sum", "mean"], ) - transformer.fit(df_vartypes) + transformer.fit(df) with pytest.raises(ValueError): transformer.get_feature_names_out(input_features=_input_features) diff --git a/tests/test_creation/test_relative_features.py b/tests/test_creation/test_relative_features.py index dbfa4972c..e8dc5971c 100644 --- a/tests/test_creation/test_relative_features.py +++ b/tests/test_creation/test_relative_features.py @@ -1,10 +1,34 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.pipeline import Pipeline from feature_engine.creation import RelativeFeatures +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + + +def _none_to_nan(values): + # Missing values print as None for polars, NaN for pandas float columns + # - both mean "missing" here, so normalize both sides before comparing. + return [np.nan if v is None else v for v in values] + + +def assert_df_equal(X, expected: dict, abs_tol: float = 1e-5) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert _none_to_nan(result[col]) == pytest.approx( + _none_to_nan(values), abs=abs_tol, nan_ok=True + ) + def test_mandatory_init_parameters(): with pytest.raises(TypeError): @@ -75,14 +99,16 @@ def test_error_when_drop_original_not_bool(): ) -def test_error_when_variables_not_numeric(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_variables_not_numeric(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Name", "Age", "Marks"], reference=["Age", "Name"], func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) transformer = RelativeFeatures( reference=["Name", "Age", "Marks"], @@ -90,17 +116,19 @@ def test_error_when_variables_not_numeric(df_vartypes): func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) -def test_error_when_entered_variables_not_in_df(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_entered_variables_not_in_df(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["FeatOutsideDataset", "Age"], reference=["Age", "Name"], func=["sub"], ) with pytest.raises(KeyError): - transformer.fit_transform(df_vartypes) + transformer.fit_transform(df) transformer = RelativeFeatures( reference=["FeatOutsideDataset", "Age"], @@ -108,146 +136,126 @@ def test_error_when_entered_variables_not_in_df(df_vartypes): func=["sub"], ) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) - + transformer.fit_transform(df) -def test_classic_binary_operation(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_classic_binary_operation(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=["sub", "div", "add", "mul"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Age_div_Marks"] = [22.22222222222222, 26.25, 27.142857142857146, 30.0] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_mul_Marks"] = [18.0, 16.8, 13.3, 10.8] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Age_div_Marks": [22.22222222222222, 26.25, 27.142857142857146, 30.0], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Age_mul_Marks": [18.0, 16.8, 13.299999999999999, 10.799999999999999], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -def test_alternative_operation(df_vartypes): + assert_df_equal(Xt, expected) - # input df - df = df_vartypes.copy() - - # Expected result - dft = df.copy() - dft["Age_truediv_Marks"] = dft["Age"].truediv(dft["Marks"]) - dft["Age_floordiv_Marks"] = dft["Age"].floordiv(dft["Marks"]) - dft["Age_mod_Marks"] = dft["Age"].mod(dft["Marks"]) - dft["Age_pow_Marks"] = dft["Age"].pow(dft["Marks"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_alternative_operation(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=["truediv", "floordiv", "mod", "pow"], ) - X = transformer.fit_transform(df) + Xt = transformer.fit_transform(df) + + expected = dict(DATA) + expected["Age_truediv_Marks"] = [22.22222222222222, 26.25, 27.142857142857146, 30.0] + expected["Age_floordiv_Marks"] = [22.0, 26.0, 27.0, 30.0] + expected["Age_mod_Marks"] = [ + 0.1999999999999995, + 0.19999999999999885, + 0.1000000000000012, + 6.661338147750939e-16, + ] + expected["Age_pow_Marks"] = [ + 14.822688982138954, + 11.42287530066645, + 7.85466234994081, + 5.664525067769412, + ] - pd.testing.assert_frame_equal(X, dft) + assert_df_equal(Xt, expected) -def test_operations_with_multiple_variables(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_operations_with_multiple_variables(make_df): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["sub"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) + expected = dict(DATA) + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) + assert_df_equal(Xt, expected) - pd.testing.assert_frame_equal(X, ref) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_multiple_operations_with_multiple_variables(make_df): + df = make_df(DATA) -def test_multiple_operations_with_multiple_variables(df_vartypes): + # column order follows func order: sub's 4 columns, then add's 4 transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["sub", "add"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - } - ) + expected = dict(DATA) + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + expected["Age_add_Age"] = [40, 42, 38, 36] + expected["Marks_add_Age"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Marks_add_Marks"] = [1.8, 1.6, 1.4, 1.2] - pd.testing.assert_frame_equal(X, ref) + assert_df_equal(Xt, expected) + # reversing func order reverses the corresponding column block order transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], ) + Xt = transformer.fit_transform(df) - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) + expected = dict(DATA) + expected["Age_add_Age"] = [40, 42, 38, 36] + expected["Marks_add_Age"] = [20.9, 21.8, 19.7, 18.6] + expected["Age_add_Marks"] = [20.9, 21.8, 19.7, 18.6] + expected["Marks_add_Marks"] = [1.8, 1.6, 1.4, 1.2] + expected["Age_sub_Age"] = [0, 0, 0, 0] + expected["Marks_sub_Age"] = [-19.1, -20.2, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, 20.2, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + assert_df_equal(Xt, expected) -def test_when_missing_values_is_ignore(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_when_missing_values_is_ignore(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) transformer = RelativeFeatures( variables=["Age", "Marks"], @@ -255,30 +263,22 @@ def test_when_missing_values_is_ignore(df_vartypes): func=["sub"], missing_values="ignore", ) + Xt = transformer.fit_transform(df_na) - X = transformer.fit_transform(df_na) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, np.nan, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="min"), - "Age_sub_Age": [0, np.nan, 0, 0], - "Marks_sub_Age": [-19.1, np.nan, -18.3, -17.4], - "Age_sub_Marks": [19.1, np.nan, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) + expected = dict(data_na) + expected["Age_sub_Age"] = [0, np.nan, 0, 0] + expected["Marks_sub_Age"] = [-19.1, np.nan, -18.3, -17.4] + expected["Age_sub_Marks"] = [19.1, np.nan, 18.3, 17.4] + expected["Marks_sub_Marks"] = [0.0, 0.0, 0.0, 0.0] + assert_df_equal(Xt, expected) -def test_error_when_null_values_in_variable(df_vartypes): - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_when_null_values_in_variable(make_df): + data_na = dict(DATA) + data_na["Age"] = [20, None, 19, 18] + df_na = make_df(data_na) transformer = RelativeFeatures( variables=["Age", "Marks"], @@ -290,14 +290,16 @@ def test_error_when_null_values_in_variable(df_vartypes): with pytest.raises(ValueError): transformer.fit(df_na) - transformer.fit(df_vartypes) + transformer.fit(make_df(DATA)) with pytest.raises(ValueError): transformer.transform(df_na) -def test_when_df_cols_are_integers(df_vartypes): - df = df_vartypes.copy() - df.columns = [0, 1, 2, 3, 4] +def test_when_df_cols_are_integers(): + # polars requires string column names, so int-named columns are + # pandas-only - no polars equivalent to parametrize against here. + df = pd.DataFrame(DATA) + df.columns = [0, 1, 2, 3] transformer = RelativeFeatures( variables=[2, 3], @@ -313,7 +315,6 @@ def test_when_df_cols_are_integers(df_vartypes): 1: ["London", "Manchester", "Liverpool", "Bristol"], 2: [20, 21, 19, 18], 3: [0.9, 0.8, 0.7, 0.6], - 4: pd.date_range("2020-02-24", periods=4, freq="min"), "2_sub_2": [0, 0, 0, 0], "3_sub_2": [-19.1, -20.2, -18.3, -17.4], "2_sub_3": [19.1, 20.2, 18.3, 17.4], @@ -328,31 +329,30 @@ def test_when_df_cols_are_integers(df_vartypes): pd.testing.assert_frame_equal(X, ref) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_func", [["div"], ["truediv"], ["floordiv"], ["mod"]]) -def test_error_when_division_by_zero_and_fill_value_is_none(_func, df_vartypes): - - df_zero = df_vartypes.copy() - df_zero.loc[1, "Marks"] = 0 +def test_error_when_division_by_zero_and_fill_value_is_none(make_df, _func): + data_zero = dict(DATA) + data_zero["Marks"] = [0.9, 0, 0.7, 0.6] + df_zero = make_df(data_zero) transformer = RelativeFeatures( variables=["Age"], reference=["Marks"], func=_func, ) - transformer.fit(df_vartypes) - - with pytest.raises(ValueError) as record: - transformer.transform(df_zero) + transformer.fit(make_df(DATA)) msg = ( "Some of the reference variables contain zeroes. Division by zero " "does not exist. Replace zeros before using this transformer for division " "or set `fill_value` to a number." ) - # check that the error message matches - assert str(record.value) == msg + with pytest.raises(ValueError, match=msg): + transformer.transform(df_zero) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "_fill_value, _func", [ @@ -366,11 +366,13 @@ def test_error_when_division_by_zero_and_fill_value_is_none(_func, df_vartypes): (999, ["mod"]), ], ) -def test_fill_values_when_division_by_zero(_fill_value, _func, df_vartypes): - df_zero = df_vartypes.copy() - df_zero.loc[2, "Marks"] = 0 - df_zero.loc[1, "Age"] = np.nan - df_zero.loc[3, "Age"] = np.inf +def test_fill_values_when_division_by_zero(make_df, _fill_value, _func): + data_zero = dict(DATA) + data_zero["Marks"] = [0.9, 0.8, 0, 0.6] + # Age must be float from the start: polars can't build an Int64 column + # from a mix of ints and NaN/inf the way pandas silently upcasts to. + data_zero["Age"] = [20.0, np.nan, 19.0, np.inf] + df_zero = make_df(data_zero) transformer = RelativeFeatures( variables=["Age"], @@ -379,18 +381,20 @@ def test_fill_values_when_division_by_zero(_fill_value, _func, df_vartypes): func=_func, missing_values="ignore", ) - - X = transformer.fit_transform(df_zero) + Xt = transformer.fit_transform(df_zero) new_var = f"Age_{_func[0]}_Marks" + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) - assert X.loc[2, new_var] == _fill_value - np.testing.assert_equal(X.loc[1, "Age"], np.nan) - np.testing.assert_equal(X.loc[3, "Age"], np.inf) + assert result[new_var][2] == pytest.approx(_fill_value) + np.testing.assert_equal(result["Age"][1], np.nan) + np.testing.assert_equal(result["Age"][3], np.inf) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out(_drop, df_vartypes): +def test_get_feature_names_out(make_df, _drop): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], @@ -407,55 +411,88 @@ def test_get_feature_names_out(_drop, df_vartypes): "Age_sub_Marks", "Marks_sub_Marks", ] - X = transformer.fit_transform(df_vartypes) - feat_out = list(X.columns) + Xt = transformer.fit_transform(df) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) assert feat_out == transformer.get_feature_names_out(input_features=None) - assert feat_out == transformer.get_feature_names_out( - input_features=df_vartypes.columns - ) assert all([f for f in varnames if f in feat_out]) + if _drop is True: + # drop_original only drops columns that are in variables/reference + # (here Age, Marks) - Name and City are neither, so they remain. + assert feat_out == ["Name", "City"] + varnames + else: + assert feat_out == list(DATA.keys()) + varnames +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out_from_pipeline(_drop, df_vartypes): +def test_get_feature_names_out_from_pipeline(make_df, _drop): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], drop_original=_drop, ) - pipe = Pipeline([("transformer", transformer)]) - varnames = [ - "Age_add_Age", - "Marks_add_Age", - "Age_add_Marks", - "Marks_add_Marks", - "Age_sub_Age", - "Marks_sub_Age", - "Age_sub_Marks", - "Marks_sub_Marks", - ] - - X = pipe.fit_transform(df_vartypes) - assert list(X.columns) == pipe.get_feature_names_out(input_features=None) - assert list(X.columns) == pipe.get_feature_names_out( - input_features=df_vartypes.columns - ) - assert all([f for f in varnames if f in X.columns]) + Xt = pipe.fit_transform(df) + feat_out = list(nw.from_native(Xt, eager_only=True).columns) + assert feat_out == pipe.get_feature_names_out(input_features=None) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) -def test_get_feature_names_out_raises_error_when_wrong_param( - _input_features, df_vartypes -): +def test_get_feature_names_out_raises_error_when_wrong_param(make_df, _input_features): + df = make_df(DATA) transformer = RelativeFeatures( variables=["Age", "Marks"], reference=["Age", "Marks"], func=["add", "sub"], ) - transformer.fit(df_vartypes) + transformer.fit(df) with pytest.raises(ValueError): transformer.get_feature_names_out(input_features=_input_features) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_mixed_int_float_variables_preserve_own_dtype(make_df): + # a regression check: extracting variables as one batched 2D array + # upcasts everything to a common dtype, losing e.g. an int column's + # own int result for subtraction. Each variable must keep its own + # dtype promotion, independent of the other variables in the list. + df = make_df(DATA) + transformer = RelativeFeatures( + variables=["Age", "Marks"], reference=["Age"], func=["sub"] + ) + Xt = transformer.fit_transform(df) + nw_Xt = nw.from_native(Xt, eager_only=True) + assert nw_Xt.get_column("Age_sub_Age").dtype.is_integer() + assert not nw_Xt.get_column("Marks_sub_Age").dtype.is_integer() + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_floordiv_zero_with_float_fill_value_widens_dtype(make_df): + # floordiv on integer input stays integer-typed; a float fill_value + # must widen the result column rather than truncating or erroring, + # matching pandas' own automatic dtype promotion here. + df = make_df({"v": [7, 8], "ref": [0, 2]}) + transformer = RelativeFeatures( + variables=["v"], reference=["ref"], func=["floordiv"], fill_value=-1.5 + ) + Xt = transformer.fit_transform(df) + result = nw.from_native(Xt, eager_only=True).get_column("v_floordiv_ref").to_list() + assert result == pytest.approx([-1.5, 4.0]) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_drop_original_both_backends(make_df): + df = make_df({"x1": [1, 2, 3], "x2": [4, 5, 6], "x3": [3, 4, 5]}) + transformer = RelativeFeatures( + variables=["x1", "x2"], reference=["x3"], func=["div"], drop_original=True + ) + Xt = transformer.fit_transform(df) + assert list(nw.from_native(Xt, eager_only=True).columns) == [ + "x1_div_x3", + "x2_div_x3", + ] diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 711b1aea7..ccef69112 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,138 +1,310 @@ import numpy as np import pandas as pd +import polars as pl import pytest from pandas.testing import assert_frame_equal, assert_series_equal +from polars.testing import assert_frame_equal as pl_assert_frame_equal +from polars.testing import assert_series_equal as pl_assert_series_equal from scipy.sparse import csr_matrix from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, - _check_optional_contains_na, _check_X_matches_training_df, check_X, check_X_y, check_y, ) +# ------------------------ +# test check_X +# ------------------------ -def test_check_X_returns_df(df_vartypes): - assert_frame_equal(check_X(df_vartypes), df_vartypes) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_unchanged(make_df, assert_equal_fn): + df = make_df({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) + X = check_X(df) + assert isinstance(X, type(df)) + assert_equal_fn(X, df) -def test_check_X_converts_numpy_to_pandas(): - a1D = np.array([1, 2, 3, 4]) - a2D = np.array([[1, 2], [3, 4]]) - a3D = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) - - df_2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - assert_frame_equal(df_2D, check_X(a2D)) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_with_mixed_dtypes(make_df, assert_equal_fn): + data = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), + } + df = make_df(data) + assert_equal_fn(check_X(df), df) + + +@pytest.mark.parametrize( + "df", + [ + pd.DataFrame([]), + pd.DataFrame({"a": []}), + pl.DataFrame({"a": []}), + ], +) +def test_raises_error_if_empty_df(df): with pytest.raises(ValueError): - check_X(a3D) + check_X(df) + + +def test_check_X_raises_error_if_0_columns(): + # A dataframe with rows but no columns is not caught by `is_empty()`, which + # only looks at the row count, so it needs its own explicit check. Polars has + # no representation for "rows with 0 columns", so this case is pandas-only. + df = pd.DataFrame(index=range(3)) + assert df.shape == (3, 0) with pytest.raises(ValueError): - check_X(a1D) + check_X(df) + + +def test_check_X_raises_error_on_duplicated_column_names(): + # only relevant for pandas + df = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + } + ) + df.columns = ["var_A", "var_A", "var_B", "var_C"] + msg = "Expected unique column names" + with pytest.raises(ValueError, match=msg): + check_X(df) -def test_check_X_raises_error_sparse_matrix(): - sparse_mx = csr_matrix([[5]]) - with pytest.raises(TypeError): - assert check_X(sparse_mx) +@pytest.mark.parametrize( + "X", + [ + np.array([[1, 2], [3, 4]]), + np.array([1, 2, 3]), + np.array(1), + [1, 2, 3], + {"a": [1, 2, 3]}, + "not a dataframe", + None, + csr_matrix([[1, 2], [3, 4]]), + ], +) +def test_check_X_raises_error_on_non_dataframe_input(X): + with pytest.raises(TypeError) as record: + check_X(X) + assert record.match("X must be a dataframe from a library supported by narwhals") -def test_check_X_raises_error_with_complex_data(): - msg = "Complex data not supported" - rng = np.random.RandomState(0) - X = rng.uniform(size=10) + 1j * rng.uniform(size=10) - X = X.reshape(-1, 1) - with pytest.raises(TypeError, match=msg): - assert check_X(X) +# ------------------------ +# test check_y +# ------------------------ +# --- series input --- -def test_raises_error_if_empty_df(): - df = pd.DataFrame([]) - with pytest.raises(ValueError): - check_X(df) + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_returns_values_unchanged(make_series, assert_equal_fn): + s = make_series([0, 1, 2, 3, 4]) + assert_equal_fn(check_y(s), s) -def test_check_y_returns_series(): - s = pd.Series([0, 1, 2, 3, 4]) - assert_series_equal(check_y(s), s) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_nan_error(make_series): + s = make_series([0.0, None, 2.0]) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(s) -def test_check_y_returns_dataframe(): - d = pd.DataFrame({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) - assert_frame_equal(check_y(d), d) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_nan_error_for_explicit_nan(make_series): + # in polars, an explicit float("nan") is not a null value, so it is only + # caught if is_nan() is checked in addition to is_null() + s = make_series([0.0, float("nan"), 2.0]) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(s) -def test_check_y_converts_np_array(): - a1D = np.array([1, 2, 3, 4]) - s = pd.Series(a1D) - assert_series_equal(check_y(a1D), s) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_inf_error(make_series): + s = make_series([0.0, float("inf"), 2.0]) + with pytest.raises(ValueError, match="y contains infinity values."): + check_y(s) -def test_check_y_converts_np_array_2D(): - a2D = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4) - d = pd.DataFrame(a2D) - assert_frame_equal(check_y(a2D), d) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_converts_string_to_number_when_y_numeric( + make_series, assert_equal_fn +): + s = make_series(["0", "1", "2"]) + y = check_y(s, y_numeric=True) + expected = make_series([0.0, 1.0, 2.0]) + assert_equal_fn(y, expected) + + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_leaves_non_numeric_unchanged_by_default( + make_series, assert_equal_fn +): + # y_numeric defaults to False: a non-numeric series (e.g. classification + # labels) should be returned as-is, without being cast to float. + s = make_series(["a", "b", "c"]) + assert_equal_fn(check_y(s), s) -def test_check_y_raises_none_error(): - with pytest.raises(ValueError): - check_y(None) +# --- dataframe (multioutput) input --- -def test_check_y_raises_nan_error(): - msg = "y contains NaN values." +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_y_dataframe_returns_values_unchanged(make_df, assert_equal_fn): + d = make_df({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) + assert_equal_fn(check_y(d), d) - # y is series - s = pd.Series([0, np.nan, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.nan, 3, 4, 5, 6, np.nan, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_nan_error(make_df): + d = make_df({"t1": [0.0, None, 2.0], "t2": [5.0, 6.0, 7.0]}) + with pytest.raises(ValueError, match="y contains NaN values."): check_y(d) - assert str(record.value) == msg -def test_check_y_raises_inf_error(): - msg = "y contains infinity values." +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_nan_error_for_explicit_nan(make_df): + # in polars, an explicit float("nan") is not a null value, so it is only + # caught if is_nan() is checked in addition to is_null() + d = make_df({"t1": [0.0, float("nan"), 2.0], "t2": [5.0, 6.0, 7.0]}) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(d) - # y is series - s = pd.Series([0, np.inf, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.inf, 3, 4, 5, 6, np.inf, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_inf_error(make_df): + d = make_df({"t1": [0.0, 0.4, 2.0], "t2": [5.0, float("inf"), 7.0]}) + with pytest.raises(ValueError, match="y contains infinity values."): check_y(d) - assert str(record.value) == msg -def test_check_y_converts_string_to_number(): - s = pd.Series(["0", "1", "2", "3", "4"]) - assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) +# --- array-like input --- -def test_check_x_y_returns_pandas_from_pandas(df_vartypes): - # when s is series - s = pd.Series([0, 1, 2, 3]) - x, y = check_X_y(df_vartypes, s) - assert_frame_equal(df_vartypes, x) - assert_series_equal(s, y) +@pytest.mark.parametrize( + "a", + [ + np.array([1, 2, 3, 4]), + np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4), + [1, 2, 3, 4], + ], +) +def test_check_y_array_returns_unchanged(a): + y = check_y(a) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a, y) + + +def test_check_y_raises_none_error(): + msg = "requires y to be passed, but the target y" + with pytest.raises(ValueError, match=msg): + check_y(None) + + +# ------------------------ +# test check_X_y +# ------------------------ - # when y is multioutput - d = pd.DataFrame(np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2)) - x, y = check_X_y(df_vartypes, d) - assert_frame_equal(df_vartypes, x) - assert_frame_equal(d, y) + +@pytest.mark.parametrize( + "make_df, assert_frame_fn, make_series, assert_series_fn", + [ + (pd.DataFrame, assert_frame_equal, pd.Series, assert_series_equal), + (pl.DataFrame, pl_assert_frame_equal, pl.Series, pl_assert_series_equal), + ], +) +def test_check_X_y_returns_df_and_series_unchanged( + make_df, assert_frame_fn, make_series, assert_series_fn +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + s = make_series([0, 1, 2]) + X, y = check_X_y(df, s) + assert isinstance(X, type(df)) and isinstance(y, type(s)) + assert_frame_fn(X, df) + assert_series_fn(y, s) + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_y_returns_df_and_multioutput_y_unchanged(make_df, assert_frame_fn): + df = make_df({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + d = make_df({"t1": [1, 2, 3, 4], "t2": [5, 6, 7, 8]}) + X, y = check_X_y(df, d) + assert_frame_fn(X, df) + assert_frame_fn(y, d) + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +@pytest.mark.parametrize( + "y", + [ + np.array([0, 1, 2]), + [0, 1, 2], + np.array([[0, 1], [2, 3], [4, 5]]), + ], +) +def test_check_X_y_with_array_like_y_returns_check_y_output( + make_df, assert_frame_fn, y +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + X, y_out = check_X_y(df, y) + assert_frame_fn(X, df) + np.testing.assert_array_equal(y_out, check_y(y)) -def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): +def test_check_X_y_returns_pandas_with_non_typical_index(): + # only relevant for pandas: polars has no index to reconcile df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) x, y = check_X_y(df, s) @@ -141,164 +313,139 @@ def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): def test_check_X_y_raises_error_when_pandas_index_dont_match(): + # only relevant for pandas: polars has no index to reconcile msg = "The indexes of X and y do not match." df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, s) - assert str(record.value) == msg # when y is multioutput d = pd.DataFrame( np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2), index=[22, 99, 101, 999] ) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, d) - assert str(record.value) == msg -def test_check_x_y_reassings_index_when_only_one_input_is_pandas(): - # X is dataframe, y is 1D array - df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) - s = np.array([1, 2, 3, 4]) - s_exp = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_series_equal(s_exp.astype(int), y.astype(int)) +@pytest.mark.parametrize( + "make_df, make_series", + [(pd.DataFrame, pd.Series), (pl.DataFrame, pl.Series)], +) +def test_check_x_y_raises_error_when_inconsistent_length(make_df, make_series): + df = make_df({"a": [1, 2, 3]}) + s = make_series([0, 1]) + with pytest.raises(ValueError): + check_X_y(df, s) - # X is dataframe, y is 2d array - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s_exp = pd.DataFrame(s, index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_frame_equal(s_exp.astype(int), y.astype(int)) - # X is not a df, y is a series - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_series_equal(s, y) +# ----------------------------------- +# test _check_X_matches_training_df +# ----------------------------------- - # X is not a df, y is a dataframe - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s = pd.DataFrame(s, index=[22, 99, 101, 212]) - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_frame_equal(s, y) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_passes_when_columns_match(make_df): + df = make_df({"a": [1, 2], "b": [3, 4]}) + assert _check_X_matches_training_df(df, 2) is None -def test_check_x_y_converts_numpy_to_pandas(): - a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) - df2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - a1D = np.array([1, 2, 3, 4]) - s1D = pd.Series(a1D) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_raises_error_when_columns_dont_match(make_df): + msg = "The number of columns in this dataset is different from" + df = make_df({"a": [1, 2], "b": [3, 4]}) + with pytest.raises(ValueError, match=msg): + _check_X_matches_training_df(df, 3) + - # X is df and y is array - x, y = check_X_y(df2D, a1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) +# ------------------------- +# test _check_contains_na +# ------------------------- - # X is array and y is series - x, y = check_X_y(a2D, s1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) - # X is df and y is 2d array - y2D = pd.DataFrame(a2D, columns=[0, 1]) - x, y = check_X_y(df2D, a2D) - assert_frame_equal(df2D, x) - assert_frame_equal(y2D, y) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_when_nan(make_df): + msg1 = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + msg2 = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) + + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + with pytest.raises(ValueError, match=msg1): + _check_contains_na(df, ["Name", "City"]) - # X is array and y multioutput df - x, y = check_X_y(a2D, df2D) - assert_frame_equal(df2D, x) - assert_frame_equal(df2D, y) + with pytest.raises(ValueError, match=msg2): + _check_contains_na(df, ["Name", "City"], error_msg="other") -def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): - s = pd.Series([0, 1, 2, 3, 5]) - with pytest.raises(ValueError): - check_X_y(df_vartypes, s) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_passes_when_no_nan(make_df): + df = make_df({"Name": ["tom", "nick"], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["Name", "City"]) is None -def test_check_X_matches_training_df(df_vartypes): - with pytest.raises(ValueError): - assert _check_X_matches_training_df(df_vartypes, 4) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_ignores_columns_not_in_variables(make_df): + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["City"]) is None -def test_contains_na(df_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_for_explicit_nan_in_numeric_column(make_df): + # in polars, an explicit float("nan") is not a null value, so it is only + # caught if is_nan() is checked in addition to is_null() msg = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) - - with pytest.raises(ValueError) as record: - assert _check_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg + df = make_df({"Age": [20.0, float("nan"), 19.0], "City": ["a", "b", "c"]}) + with pytest.raises(ValueError, match=msg): + _check_contains_na(df, ["Age", "City"]) -def test_optional_contains_na(df_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_for_mix_of_null_and_nan_across_dtypes(make_df): + # a numeric column with a NaN and a string column with a null should both + # still be caught, and the numeric-only is_nan() scoping must not error out + # on the string column msg = ( "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." + "remove those before using this transformer." ) + df = make_df({"Age": [20.0, float("nan"), 19.0], "City": ["a", None, "c"]}) + with pytest.raises(ValueError, match=msg): + _check_contains_na(df, ["Age", "City"]) - with pytest.raises(ValueError) as record: - assert _check_optional_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg +# -------------------------- +# test _check_contains_inf +# -------------------------- -def test_contains_inf_raises_on_inf(): + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_raises_on_inf(make_df): msg = ( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) - df = pd.DataFrame({"A": [1.1, np.inf, 3.3]}) + df = make_df({"A": [1.1, np.inf, 3.3]}) with pytest.raises(ValueError, match=msg): _check_contains_inf(df, ["A"]) -def test_contains_inf_passes_without_inf(): - df = pd.DataFrame({"A": [1.1, 2.2, 3.3]}) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_passes_without_inf(make_df): + df = make_df({"A": [1.1, 2.2, 3.3]}) assert _check_contains_inf(df, ["A"]) is None -def test_check_X_raises_error_on_duplicated_column_names(): - df = pd.DataFrame( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - } - ) - df.columns = ["var_A", "var_A", "var_B", "var_C"] - with pytest.raises(ValueError) as err_txt: - check_X(df) - assert err_txt.match("Input data contains duplicated variable names.") - - -def test_check_X_errors(): - # Test scalar array error (line 58) - with pytest.raises(ValueError) as record: - check_X(np.array(1)) - assert record.match("Expected 2D array, got scalar array instead") - - # Test 1D array error (line 65) - with pytest.raises(ValueError) as record: - check_X(np.array([1, 2, 3])) - assert record.match("Expected 2D array, got 1D array instead") - - # Test incorrect type error (line 80) - with pytest.raises(TypeError) as record: - check_X("not a dataframe") - assert record.match("X must be a numpy array or pandas dataframe") +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_ignores_columns_not_in_variables(make_df): + df = make_df({"A": [1.1, float("inf"), 3.3], "B": [1.0, 2.0, 3.0]}) + assert _check_contains_inf(df, ["B"]) is None diff --git a/tests/test_discretisation/test_check_estimator_discretisers.py b/tests/test_discretisation/test_check_estimator_discretisers.py index a1f78c1e0..2c7e1a332 100644 --- a/tests/test_discretisation/test_check_estimator_discretisers.py +++ b/tests/test_discretisation/test_check_estimator_discretisers.py @@ -1,10 +1,8 @@ import numpy as np import pandas as pd import pytest -import sklearn from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.discretisation import ( ArbitraryDiscretiser, @@ -18,9 +16,6 @@ check_raises_non_fitted_error_when_fit_fails, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - - _estimators = [ DecisionTreeDiscretiser(regression=False), EqualFrequencyDiscretiser(), @@ -29,20 +24,13 @@ GeometricWidthDiscretiser(), ] -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) -else: - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator( - estimator=estimator, - expected_failed_checks=estimator._more_tags()["_xfail_checks"], - ) +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + return check_estimator( + estimator=estimator, + expected_failed_checks=estimator._more_tags()["_xfail_checks"], + ) @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 0e30f2939..82e299588 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,12 +1,10 @@ import pandas as pd import pytest -import sklearn from numpy import nan from sklearn import clone from sklearn.exceptions import NotFittedError from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.encoding import ( CountEncoder, @@ -25,8 +23,6 @@ test_df, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - _estimators = [ CountEncoder(ignore_format=True), CountFrequencyEncoder(ignore_format=True), @@ -46,22 +42,16 @@ ] -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) +expected_fails = _return_tags()["_xfail_checks"] +expected_fails.update({"check_estimators_nan_inf": "transformer allows NA"}) -else: - expected_fails = _return_tags()["_xfail_checks"] - expected_fails.update({"check_estimators_nan_inf": "transformer allows NA"}) - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - if estimator.__class__.__name__ != "WoEEncoder": - return check_estimator( - estimator=estimator, expected_failed_checks=expected_fails - ) +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + if estimator.__class__.__name__ != "WoEEncoder": + return check_estimator( + estimator=estimator, expected_failed_checks=expected_fails + ) _estimators = [ diff --git a/tests/test_imputation/test_check_estimator_imputers.py b/tests/test_imputation/test_check_estimator_imputers.py index 6f9d0c4fc..3d22230f8 100644 --- a/tests/test_imputation/test_check_estimator_imputers.py +++ b/tests/test_imputation/test_check_estimator_imputers.py @@ -1,9 +1,7 @@ import pandas as pd import pytest -import sklearn from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.imputation import ( MissingIndicator, @@ -29,22 +27,13 @@ DropMissingData(), ] -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator( - estimator=estimator, - expected_failed_checks=estimator._more_tags()["_xfail_checks"], - ) +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + return check_estimator( + estimator=estimator, + expected_failed_checks=estimator._more_tags()["_xfail_checks"], + ) @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_outliers/test_check_estimator_outliers.py b/tests/test_outliers/test_check_estimator_outliers.py index c0d30300f..0b5ee3491 100644 --- a/tests/test_outliers/test_check_estimator_outliers.py +++ b/tests/test_outliers/test_check_estimator_outliers.py @@ -1,9 +1,7 @@ import pandas as pd import pytest -import sklearn from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.outliers import ArbitraryOutlierCapper, OutlierTrimmer, Winsoriser from feature_engine.tags import _return_tags @@ -15,42 +13,32 @@ Winsoriser(), ] -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - FAILED_CHECKS = _return_tags()["_xfail_checks"] - FAILED_CHECKS_AOC = _return_tags()["_xfail_checks"] - - msg1 = ( - "transformers raise errors when data variation is low, " "thus this check fails" - ) - - msg2 = "transformer has 1 mandatory parameter" - - FAILED_CHECKS.update({"check_fit2d_1sample": msg1}) - FAILED_CHECKS_AOC.update( - { - "check_fit2d_1sample": msg1, - "check_parameters_default_constructible": msg2, - } - ) - - @pytest.mark.parametrize( - "estimator, failed_tests", - [ - (_estimators[0], FAILED_CHECKS_AOC), - (_estimators[1], FAILED_CHECKS), - (_estimators[2], FAILED_CHECKS), - ], - ) - def test_check_estimator_from_sklearn(estimator, failed_tests): - return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) +FAILED_CHECKS = _return_tags()["_xfail_checks"] +FAILED_CHECKS_AOC = _return_tags()["_xfail_checks"] + +msg1 = "transformers raise errors when data variation is low, " "thus this check fails" + +msg2 = "transformer has 1 mandatory parameter" + +FAILED_CHECKS.update({"check_fit2d_1sample": msg1}) +FAILED_CHECKS_AOC.update( + { + "check_fit2d_1sample": msg1, + "check_parameters_default_constructible": msg2, + } +) + + +@pytest.mark.parametrize( + "estimator, failed_tests", + [ + (_estimators[0], FAILED_CHECKS_AOC), + (_estimators[1], FAILED_CHECKS), + (_estimators[2], FAILED_CHECKS), + ], +) +def test_check_estimator_from_sklearn(estimator, failed_tests): + return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_prediction/test_check_estimator_prediction.py b/tests/test_prediction/test_check_estimator_prediction.py index 3618933b3..afe45db71 100644 --- a/tests/test_prediction/test_check_estimator_prediction.py +++ b/tests/test_prediction/test_check_estimator_prediction.py @@ -1,11 +1,8 @@ import numpy as np import pandas as pd import pytest -import sklearn from sklearn.base import clone from sklearn.exceptions import NotFittedError -from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine._prediction.base_predictor import BaseTargetMeanEstimator from feature_engine._prediction.target_mean_classifier import TargetMeanClassifier @@ -18,18 +15,14 @@ from tests.estimator_checks.dataframe_for_checks import test_df from tests.estimator_checks.fit_functionality_checks import check_error_if_y_not_passed -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - _estimators = [BaseTargetMeanEstimator(), TargetMeanClassifier(), TargetMeanRegressor()] _predictors = [TargetMeanRegressor(), TargetMeanClassifier()] -if sklearn_version < parse_version("1.6"): - # In sklearn version 1.6, changes into the developer api were introduced - # that break the tests. Need to dig further into it. - # TODO: add tests for sklearn version > 1.6 - @pytest.mark.parametrize("estimator", [BaseTargetMeanEstimator()]) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) + +# TODO: no test_check_estimator_from_sklearn exists for this module — the previous +# sklearn<1.6 version of this test was removed when dropping sklearn<=1.6 support, +# and a sklearn>=1.6-compatible replacement (using expected_failed_checks=...) was +# never written. See the module's git history for the removed sklearn<1.6 branch. @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_preprocessing/test_check_estimator_preprocessing.py b/tests/test_preprocessing/test_check_estimator_preprocessing.py index 378091840..ca16f8863 100644 --- a/tests/test_preprocessing/test_check_estimator_preprocessing.py +++ b/tests/test_preprocessing/test_check_estimator_preprocessing.py @@ -1,12 +1,10 @@ import pandas as pd import pytest -import sklearn from numpy import nan from sklearn import clone from sklearn.exceptions import NotFittedError from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.preprocessing import MatchCategories, MatchVariables from feature_engine.tags import _return_tags @@ -15,43 +13,35 @@ test_df, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - _estimators = [MatchCategories(ignore_format=True), MatchVariables()] -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - FAILED_CHECKS = _return_tags()["_xfail_checks"] - FAILED_CHECKS_MATCHCOLS = _return_tags()["_xfail_checks"] - - msg1 = "input shape of dataframes in fit and transform can differ" - msg2 = ( - "transformer takes categorical variables, and inf cannot be determined" - "on these variables. Thus, check is not implemented" - ) - - FAILED_CHECKS.update({"check_estimators_nan_inf": msg2}) - FAILED_CHECKS_MATCHCOLS.update( - { - "check_transformer_general": msg1, - "check_estimators_nan_inf": msg2, - } - ) - - @pytest.mark.parametrize( - "estimator, failed_tests", - [ - (_estimators[0], FAILED_CHECKS), - (_estimators[1], FAILED_CHECKS_MATCHCOLS), - ], - ) - def test_check_estimator_from_sklearn(estimator, failed_tests): - return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) +FAILED_CHECKS = _return_tags()["_xfail_checks"] +FAILED_CHECKS_MATCHCOLS = _return_tags()["_xfail_checks"] + +msg1 = "input shape of dataframes in fit and transform can differ" +msg2 = ( + "transformer takes categorical variables, and inf cannot be determined" + "on these variables. Thus, check is not implemented" +) + +FAILED_CHECKS.update({"check_estimators_nan_inf": msg2}) +FAILED_CHECKS_MATCHCOLS.update( + { + "check_transformer_general": msg1, + "check_estimators_nan_inf": msg2, + } +) + + +@pytest.mark.parametrize( + "estimator, failed_tests", + [ + (_estimators[0], FAILED_CHECKS), + (_estimators[1], FAILED_CHECKS_MATCHCOLS), + ], +) +def test_check_estimator_from_sklearn(estimator, failed_tests): + return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) @pytest.mark.parametrize("estimator", [MatchCategories(), MatchVariables()]) diff --git a/tests/test_selection/test_check_estimator_selectors.py b/tests/test_selection/test_check_estimator_selectors.py index debbe165e..7ce85634b 100644 --- a/tests/test_selection/test_check_estimator_selectors.py +++ b/tests/test_selection/test_check_estimator_selectors.py @@ -1,10 +1,8 @@ import pandas as pd import pytest -import sklearn from sklearn.linear_model import LogisticRegression from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.selection import ( MRMR, @@ -29,8 +27,6 @@ check_raises_error_if_only_1_variable, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - _logreg = LogisticRegression(C=0.0001, max_iter=2, random_state=1) _estimators = [ @@ -84,27 +80,17 @@ ProbeFeatureSelection(estimator=_logreg, scoring="accuracy"), ] -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - # In sklearn 1.6. the API changes break the tests for the target mean selector. - # We need to investigate further. - # TODO: investigate checks for target mean selector. - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - if estimator.__class__.__name__ not in [ - "SelectByTargetEncoding", - "SelectByTargetMeanPerformance", - "SelectByInformationValue", - ]: - failed_tests = estimator._more_tags()["_xfail_checks"] - return check_estimator( - estimator=estimator, expected_failed_checks=failed_tests - ) + +# TODO: investigate checks for target mean selector. +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + if estimator.__class__.__name__ not in [ + "SelectByTargetEncoding", + "SelectByTargetMeanPerformance", + "SelectByInformationValue", + ]: + failed_tests = estimator._more_tags()["_xfail_checks"] + return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) @pytest.mark.parametrize("estimator", _univariate_estimators) diff --git a/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py b/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py index f9905a4d0..85a4af38c 100644 --- a/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py +++ b/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py @@ -1,11 +1,9 @@ import numpy as np import pandas as pd import pytest -import sklearn from sklearn.base import clone from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.timeseries.forecasting import ( ExpandingWindowFeatures, @@ -21,28 +19,19 @@ ] -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - extra_failing_checks = { - "check_estimators_nan_inf": "Time Series transformers do not handle NaNs " - "or infinity." - } - return check_estimator( - estimator=estimator, - expected_failed_checks={ - **extra_failing_checks, - **estimator._more_tags()["_xfail_checks"], - }, - ) +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + extra_failing_checks = { + "check_estimators_nan_inf": "Time Series transformers do not handle NaNs " + "or infinity." + } + return check_estimator( + estimator=estimator, + expected_failed_checks={ + **extra_failing_checks, + **estimator._more_tags()["_xfail_checks"], + }, + ) @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_transformation/test_arcsin_transformer.py b/tests/test_transformation/test_arcsin_transformer.py index e476c27d5..b8a161132 100644 --- a/tests/test_transformation/test_arcsin_transformer.py +++ b/tests/test_transformation/test_arcsin_transformer.py @@ -1,68 +1,83 @@ +import narwhals as nw +import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import ArcsinTransformer - -def test_transform_and_inverse_transform(df_vartypes): +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_and_inverse_transform(make_df): + X = make_df(DATA) transformer = ArcsinTransformer(variables=["Marks"]) - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Marks"] = [1.24905, 1.10715, 0.99116, 0.88607] - - # test transform output - pd.testing.assert_frame_equal(X, transf_df) - - # test inverse_transform - Xit = transformer.inverse_transform(X) + Xt = transformer.fit_transform(X) - # convert numbers to original format. - Xit["Marks"] = Xit["Marks"].round(1) + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) + assert result["Marks"] == pytest.approx( + [1.24905, 1.10715, 0.99116, 0.88607], abs=1e-5 + ) - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) + Xit = transformer.inverse_transform(Xt) + result_it = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] -def test_fit_raises_error_if_na_in_df(df_na): - # test case 2: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) transformer = ArcsinTransformer(variables=["Marks"]) with pytest.raises(ValueError): - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 3: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) transformer = ArcsinTransformer(variables=["Marks"]) - transformer.fit(df_vartypes) + transformer.fit(X) with pytest.raises(ValueError): - transformer.transform(df_na[df_vartypes.columns]) + transformer.transform(X_na) -def test_error_if_df_contains_outside_range_values(df_vartypes): - # test error when data contains value outside range [0, +1] - df_out_range = df_vartypes.copy() - df_out_range.loc[1, "Marks"] = 2 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_df_contains_outside_range_values(make_df): + data_out_range = dict(DATA) + data_out_range["Marks"] = [0.9, 2, 0.7, 0.6] + X = make_df(DATA) + X_out_range = make_df(data_out_range) transformer = ArcsinTransformer(variables=["Marks"]) - # test case 4: when variable contains value outside range, fit with pytest.raises(ValueError): - transformer.fit(df_out_range) + transformer.fit(X_out_range) - # test case 5: when variable contains value outside range, transform - transformer.fit(df_vartypes) + transformer.fit(X) with pytest.raises(ValueError): - transformer.transform(df_out_range) + transformer.transform(X_out_range) - # when selecting variables automatically and some are outside range transformer = ArcsinTransformer() with pytest.raises(ValueError): - transformer.fit(df_vartypes) + transformer.fit(X_out_range) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) transformer = ArcsinTransformer(variables="Marks") with pytest.raises(NotFittedError): - transformer.transform(df_vartypes) + transformer.transform(X) diff --git a/tests/test_transformation/test_arcsinh.py b/tests/test_transformation/test_arcsinh.py index a3d8b8d4d..aa90b10af 100644 --- a/tests/test_transformation/test_arcsinh.py +++ b/tests/test_transformation/test_arcsinh.py @@ -1,128 +1,142 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.transformation import ArcSinhTransformer +DATA_NUMERICAL = { + "a": [-100.0, -10.0, 0.0, 10.0, 100.0], + "b": [1.0, 2.0, 3.0, 4.0, 5.0], +} +DATA_MULTI_COLUMN = { + "a": [1.0, 2.0, 3.0], + "b": [4.0, 5.0, 6.0], + "c": [7.0, 8.0, 9.0], +} -@pytest.fixture -def df_numerical(): - """Fixture providing sample numerical data with positive and negative values.""" - return pd.DataFrame({ - "a": [-100, -10, 0, 10, 100], - "b": [1, 2, 3, 4, 5], - }) +def _col(X, name): + return nw.from_native(X, eager_only=True).get_column(name).to_numpy() -@pytest.fixture -def df_multi_column(): - """Fixture providing DataFrame with multiple columns.""" - return pd.DataFrame({ - "a": [1, 2, 3], - "b": [4, 5, 6], - "c": [7, 8, 9], - }) - -def test_default_parameters(df_numerical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_default_parameters(make_df): """Test transformer with default parameters applies arcsinh to all columns.""" + X = make_df(DATA_NUMERICAL) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(df_numerical.copy()) + X_tr = transformer.fit_transform(X) - expected_a = np.arcsinh(df_numerical["a"]) - expected_b = np.arcsinh(df_numerical["b"]) - np.testing.assert_array_almost_equal(X_tr["a"], expected_a) - np.testing.assert_array_almost_equal(X_tr["b"], expected_b) + expected_a = np.arcsinh(np.array(DATA_NUMERICAL["a"])) + expected_b = np.arcsinh(np.array(DATA_NUMERICAL["b"])) + np.testing.assert_array_almost_equal(_col(X_tr, "a"), expected_a) + np.testing.assert_array_almost_equal(_col(X_tr, "b"), expected_b) -def test_specific_variables(df_multi_column): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_specific_variables(make_df): """Test transformer with specific variables selected.""" + X = make_df(DATA_MULTI_COLUMN) transformer = ArcSinhTransformer(variables=["a", "b"]) - X_tr = transformer.fit_transform(df_multi_column.copy()) + X_tr = transformer.fit_transform(X) np.testing.assert_array_almost_equal( - X_tr["a"], np.arcsinh(df_multi_column["a"]) + _col(X_tr, "a"), np.arcsinh(np.array(DATA_MULTI_COLUMN["a"])) ) np.testing.assert_array_almost_equal( - X_tr["b"], np.arcsinh(df_multi_column["b"]) + _col(X_tr, "b"), np.arcsinh(np.array(DATA_MULTI_COLUMN["b"])) ) - np.testing.assert_array_equal(X_tr["c"], df_multi_column["c"]) + np.testing.assert_array_equal(_col(X_tr, "c"), np.array(DATA_MULTI_COLUMN["c"])) -def test_with_loc_and_scale(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_with_loc_and_scale(make_df): """Test transformer with loc and scale parameters.""" - X = pd.DataFrame({"a": [10, 20, 30, 40, 50]}) + data = {"a": [10.0, 20.0, 30.0, 40.0, 50.0]} + X = make_df(data) loc = 30.0 scale = 10.0 transformer = ArcSinhTransformer(loc=loc, scale=scale) - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - expected = np.arcsinh((X["a"] - loc) / scale) - np.testing.assert_array_almost_equal(X_tr["a"], expected) - np.testing.assert_almost_equal(X_tr["a"].iloc[2], 0.0, decimal=10) + expected = np.arcsinh((np.array(data["a"]) - loc) / scale) + np.testing.assert_array_almost_equal(_col(X_tr, "a"), expected) + np.testing.assert_almost_equal(_col(X_tr, "a")[2], 0.0, decimal=10) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("loc", [0.0, 10.0, -10.0, 100.5]) -def test_various_loc_values(loc): +def test_various_loc_values(make_df, loc): """Test that various loc values work correctly.""" - X = pd.DataFrame({"a": [1, 2, 3, 4, 5]}) + data = {"a": [1.0, 2.0, 3.0, 4.0, 5.0]} + X = make_df(data) transformer = ArcSinhTransformer(loc=loc) - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - expected = np.arcsinh((X["a"] - loc) / 1.0) - np.testing.assert_array_almost_equal(X_tr["a"], expected) + expected = np.arcsinh((np.array(data["a"]) - loc) / 1.0) + np.testing.assert_array_almost_equal(_col(X_tr, "a"), expected) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("scale", [0.5, 1.0, 2.0, 10.0, 100.0]) -def test_various_scale_values(scale): +def test_various_scale_values(make_df, scale): """Test that various scale values work correctly.""" - X = pd.DataFrame({"a": [1, 2, 3, 4, 5]}) + data = {"a": [1.0, 2.0, 3.0, 4.0, 5.0]} + X = make_df(data) transformer = ArcSinhTransformer(scale=scale) - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - expected = np.arcsinh((X["a"] - 0.0) / scale) - np.testing.assert_array_almost_equal(X_tr["a"], expected) + expected = np.arcsinh((np.array(data["a"]) - 0.0) / scale) + np.testing.assert_array_almost_equal(_col(X_tr, "a"), expected) -def test_inverse_transform(df_numerical): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform(make_df): """Test inverse_transform returns original values.""" - X_original = df_numerical.copy() + X = make_df(DATA_NUMERICAL) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(df_numerical.copy()) + X_tr = transformer.fit_transform(X) X_inv = transformer.inverse_transform(X_tr) - np.testing.assert_array_almost_equal(X_inv["a"], X_original["a"], decimal=10) - np.testing.assert_array_almost_equal(X_inv["b"], X_original["b"], decimal=10) + np.testing.assert_array_almost_equal( + _col(X_inv, "a"), np.array(DATA_NUMERICAL["a"]), decimal=10 + ) + np.testing.assert_array_almost_equal( + _col(X_inv, "b"), np.array(DATA_NUMERICAL["b"]), decimal=10 + ) -def test_inverse_transform_with_loc_scale(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform_with_loc_scale(make_df): """Test inverse_transform with loc and scale parameters.""" - X = pd.DataFrame({"a": [10, 20, 30, 40, 50]}) - X_original = X.copy() + data = {"a": [10.0, 20.0, 30.0, 40.0, 50.0]} + X = make_df(data) transformer = ArcSinhTransformer(loc=25.0, scale=5.0) - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) X_inv = transformer.inverse_transform(X_tr) - np.testing.assert_array_almost_equal(X_inv["a"], X_original["a"], decimal=10) + np.testing.assert_array_almost_equal( + _col(X_inv, "a"), np.array(data["a"]), decimal=10 + ) -def test_negative_values(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_negative_values(make_df): """Test that transformer handles negative values correctly.""" - X = pd.DataFrame({"a": [-1000, -500, 0, 500, 1000]}) + data = {"a": [-1000.0, -500.0, 0.0, 500.0, 1000.0]} + X = make_df(data) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) # Expected values: arcsinh([ -1000, -500, 0, 500, 1000 ]) expected = [-7.600902, -6.907755, 0.0, 6.907755, 7.600902] - np.testing.assert_array_almost_equal(X_tr["a"], expected, decimal=5) + result = _col(X_tr, "a") + np.testing.assert_array_almost_equal(result, expected, decimal=5) # Verify symmetry property: arcsinh(-x) = -arcsinh(x) - np.testing.assert_almost_equal( - X_tr["a"].iloc[0], -X_tr["a"].iloc[4], decimal=10 - ) - np.testing.assert_almost_equal( - X_tr["a"].iloc[1], -X_tr["a"].iloc[3], decimal=10 - ) + np.testing.assert_almost_equal(result[0], -result[4], decimal=10) + np.testing.assert_almost_equal(result[1], -result[3], decimal=10) @pytest.mark.parametrize("invalid_scale", [0, -1, -0.5, -100, "string", False]) @@ -139,9 +153,10 @@ def test_invalid_loc_raises_error(invalid_loc): ArcSinhTransformer(loc=invalid_loc) -def test_fit_stores_attributes(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_stores_attributes(make_df): """Test that fit stores expected attributes with correct values.""" - X = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + X = make_df({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]}) transformer = ArcSinhTransformer() transformer.fit(X) @@ -153,9 +168,10 @@ def test_fit_stores_attributes(): assert transformer.feature_names_in_ == ["a", "b"] -def test_get_feature_names_out(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out(make_df): """Test get_feature_names_out returns correct feature names.""" - X = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + X = make_df({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]}) transformer = ArcSinhTransformer() transformer.fit(X) @@ -163,9 +179,10 @@ def test_get_feature_names_out(): assert feature_names == ["a", "b"] -def test_get_feature_names_out_with_subset(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_with_subset(make_df): """Test get_feature_names_out with subset of variables.""" - X = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + X = make_df({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0], "c": [7.0, 8.0, 9.0]}) transformer = ArcSinhTransformer(variables=["a"]) transformer.fit(X) @@ -173,29 +190,36 @@ def test_get_feature_names_out_with_subset(): assert feature_names == ["a", "b", "c"] -def test_behavior_like_log_for_large_values(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_behavior_like_log_for_large_values(make_df): """Test that arcsinh behaves like log for large positive values.""" - X = pd.DataFrame({"a": [1000, 10000, 100000]}) + data = {"a": [1000.0, 10000.0, 100000.0]} + X = make_df(data) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - log_approx = np.log(2 * X["a"]) - np.testing.assert_array_almost_equal(X_tr["a"], log_approx, decimal=1) + log_approx = np.log(2 * np.array(data["a"])) + np.testing.assert_array_almost_equal(_col(X_tr, "a"), log_approx, decimal=1) -def test_behavior_like_identity_for_small_values(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_behavior_like_identity_for_small_values(make_df): """Test that arcsinh behaves like identity for small values.""" - X = pd.DataFrame({"a": [0.001, 0.01, 0.1]}) + data = {"a": [0.001, 0.01, 0.1]} + X = make_df(data) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - np.testing.assert_array_almost_equal(X_tr["a"], X["a"], decimal=2) + np.testing.assert_array_almost_equal( + _col(X_tr, "a"), np.array(data["a"]), decimal=2 + ) -def test_zero_input_returns_zero(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_zero_input_returns_zero(make_df): """Test that arcsinh(0) = 0.""" - X = pd.DataFrame({"a": [0.0]}) + X = make_df({"a": [0.0]}) transformer = ArcSinhTransformer() - X_tr = transformer.fit_transform(X.copy()) + X_tr = transformer.fit_transform(X) - assert X_tr["a"].iloc[0] == 0.0 + assert _col(X_tr, "a")[0] == 0.0 diff --git a/tests/test_transformation/test_boxcox_transformer.py b/tests/test_transformation/test_boxcox_transformer.py index 25fd20c40..172ea03f6 100644 --- a/tests/test_transformation/test_boxcox_transformer.py +++ b/tests/test_transformation/test_boxcox_transformer.py @@ -1,72 +1,98 @@ +import narwhals as nw import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import BoxCoxTransformer +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} -def test_automatically_finds_variables(df_vartypes): - # test case 1: automatically select variables - transformer = BoxCoxTransformer(variables=None) - X = transformer.fit_transform(df_vartypes) +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, None, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + + +def assert_df_equal(X, expected: dict, abs_tol: float = 1e-5) -> None: + result = nw.from_native(X, eager_only=True).to_dict(as_series=False) + assert list(result.keys()) == list(expected.keys()) + for col, values in expected.items(): + assert result[col] == pytest.approx(values, abs=abs_tol, nan_ok=True) - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [9.78731, 10.1666, 9.40189, 9.0099] - transf_df["Marks"] = [-0.101687, -0.207092, -0.316843, -0.431788] + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_finds_variables_and_inverse_transform(make_df): + df = make_df(DATA) + + transformer = BoxCoxTransformer(variables=None) + X = transformer.fit_transform(df) # test init params assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 - # test transform output - pd.testing.assert_frame_equal(X, transf_df) + assert transformer.n_features_in_ == 4 + + expected = dict(DATA) + expected["Age"] = [9.78731, 10.1666, 9.40189, 9.0099] + expected["Marks"] = [-0.101687, -0.207092, -0.316843, -0.431788] + assert_df_equal(X, expected) # test inverse_transform Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - Xit["Marks"] = Xit["Marks"].round(1) - - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) + result = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) + assert [round(v) for v in result["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result["Marks"]] == DATA["Marks"] -def test_fit_raises_error_if_df_contains_na(df_na): - # test case 2: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_df_contains_na(make_df): + df_na = make_df(DATA_NA) transformer = BoxCoxTransformer() with pytest.raises(ValueError): transformer.fit(df_na) -def test_transform_raises_error_if_df_contains_na(df_vartypes, df_na): - # test case 3: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_df_contains_na(make_df): + df = make_df(DATA) + df_na = make_df(DATA_NA) transformer = BoxCoxTransformer() - transformer.fit(df_vartypes) + transformer.fit(df) with pytest.raises(ValueError): - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(df_na) -def test_error_if_df_contains_negative_values(df_vartypes): - # test error when data contains negative values - df_neg = df_vartypes.copy() - df_neg.loc[1, "Age"] = -1 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_df_contains_negative_values(make_df): + data_neg = {k: list(v) for k, v in DATA.items()} + data_neg["Age"][1] = -1 + df_neg = make_df(data_neg) + df = make_df(DATA) - # test case 4: when variable contains negative value, fit + # when variable contains negative value, fit transformer = BoxCoxTransformer() with pytest.raises(ValueError): transformer.fit(df_neg) - # test case 5: when variable contains negative value, transform + # when variable contains negative value, transform transformer = BoxCoxTransformer() - transformer.fit(df_vartypes) + transformer.fit(df) with pytest.raises(ValueError): transformer.transform(df_neg) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + df = make_df(DATA) transformer = BoxCoxTransformer() with pytest.raises(NotFittedError): - transformer.transform(df_vartypes) + transformer.transform(df) diff --git a/tests/test_transformation/test_check_estimator_transformers.py b/tests/test_transformation/test_check_estimator_transformers.py index 6aac49791..8510c4fec 100644 --- a/tests/test_transformation/test_check_estimator_transformers.py +++ b/tests/test_transformation/test_check_estimator_transformers.py @@ -1,9 +1,7 @@ import pandas as pd import pytest -import sklearn from sklearn.pipeline import Pipeline from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.transformation import ( ArcsinTransformer, @@ -31,53 +29,45 @@ YeoJohnsonTransformer(), ] -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) - -if sklearn_version < parse_version("1.6"): - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) - -else: - checks_with_negative_values = [ - "check_readonly_memmap_input", - "check_fit_score_takes_y", - "check_dont_overwrite_parameters", - "check_estimators_nan_inf", - "check_f_contiguous_array_estimator", - "check_fit2d_1feature", - "check_fit2d_1sample", - "check_dict_unchanged", - "check_fit_check_is_fitted", - "check_n_features_in", - "check_positive_only_tag_during_fit", - "check_methods_subset_invariance", - ] - estimators_not_supporting_negative_values = [ - "BoxCoxTransformer", - "LogTransformer", - "ArcsinTransformer", - ] - extra_failing_checks = { - estimator_name: dict.fromkeys( - checks_with_negative_values, - "this checks passes a negative value which is not supported by " - "the transformer", - ) - for estimator_name in estimators_not_supporting_negative_values - } - - @pytest.mark.parametrize("estimator", _estimators) - def test_check_estimator_from_sklearn(estimator): - expected_failed_checks = estimator._more_tags()["_xfail_checks"] - expected_failed_checks.update( - extra_failing_checks.get(estimator.__class__.__name__, {}) - ) - return check_estimator( - estimator=estimator, - expected_failed_checks=expected_failed_checks, - ) +checks_with_negative_values = [ + "check_readonly_memmap_input", + "check_fit_score_takes_y", + "check_dont_overwrite_parameters", + "check_estimators_nan_inf", + "check_f_contiguous_array_estimator", + "check_fit2d_1feature", + "check_fit2d_1sample", + "check_dict_unchanged", + "check_fit_check_is_fitted", + "check_n_features_in", + "check_positive_only_tag_during_fit", + "check_methods_subset_invariance", +] +estimators_not_supporting_negative_values = [ + "BoxCoxTransformer", + "LogTransformer", + "ArcsinTransformer", +] +extra_failing_checks = { + estimator_name: dict.fromkeys( + checks_with_negative_values, + "this checks passes a negative value which is not supported by " + "the transformer", + ) + for estimator_name in estimators_not_supporting_negative_values +} + + +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + expected_failed_checks = estimator._more_tags()["_xfail_checks"] + expected_failed_checks.update( + extra_failing_checks.get(estimator.__class__.__name__, {}) + ) + return check_estimator( + estimator=estimator, + expected_failed_checks=expected_failed_checks, + ) @pytest.mark.parametrize("estimator", _estimators[4:]) diff --git a/tests/test_transformation/test_log_transformer.py b/tests/test_transformation/test_log_transformer.py index 23a74104c..757becf5d 100644 --- a/tests/test_transformation/test_log_transformer.py +++ b/tests/test_transformation/test_log_transformer.py @@ -1,175 +1,195 @@ +import re + +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import LogTransformer - -def test_transforming_int_vars(): - df = pd.DataFrame( - { - "var1": [1, 2, 3], - "var2": [4, 5, 3], - } - ) - dft = np.log(df) +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} +DATA_C = { + "vara": [0, 1, 2, 3], + "varb": [5, 5, 6, 7], + "varc": [-2, -1, 0, 4], + "vard": [-3, -2, -1, -5], + "vare": ["a", "b", "c", "d"], +} +DATA_C_VARS = ["vara", "varb", "varc", "vard"] +DATA_C_AUTO = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} + + +def _to_dict(X): + return nw.from_native(X, eager_only=True).to_dict(as_series=False) + + +def _expected_log(c, base): + fn = np.log if base == "e" else np.log10 + out = {} + for var in DATA_C_VARS: + c_var = c[var] if isinstance(c, dict) else c + out[var] = [fn(x + c_var) for x in DATA_C[var]] + return out + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transforming_int_vars(make_df): + X = make_df({"var1": [1, 2, 3], "var2": [4, 5, 3]}) transformer = LogTransformer(base="e", variables=None) - X = transformer.fit_transform(df) - pd.testing.assert_frame_equal(X, dft) + Xt = transformer.fit_transform(X) + result = _to_dict(Xt) + assert result["var1"] == pytest.approx(list(np.log([1, 2, 3]))) + assert result["var2"] == pytest.approx(list(np.log([4, 5, 3]))) -def test_log_base_e_plus_automatically_find_variables(df_vartypes): - # test case 1: log base e, automatically select variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_base_e_plus_automatically_find_variables(make_df): + X = make_df(DATA) transformer = LogTransformer(base="e", variables=None) - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [2.99573, 3.04452, 2.94444, 2.89037] - transf_df["Marks"] = [-0.105361, -0.223144, -0.356675, -0.510826] + Xt = transformer.fit_transform(X) # test init params assert transformer.base == "e" assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = _to_dict(Xt) + assert result["Age"] == pytest.approx( + [2.99573, 3.04452, 2.94444, 2.89037], abs=1e-5 + ) + assert result["Marks"] == pytest.approx( + [-0.105361, -0.223144, -0.356675, -0.510826], abs=1e-5 + ) # test inverse_transform - Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - Xit["Marks"] = Xit["Marks"].round(1) + Xit = transformer.inverse_transform(Xt) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) - -def test_log_base_10_plus_user_passes_var_list(df_vartypes): - # test case 2: log base 10, user passes variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_base_10_plus_user_passes_var_list(make_df): + X = make_df(DATA) transformer = LogTransformer(base="10", variables="Age") - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [1.30103, 1.32222, 1.27875, 1.25527] + Xt = transformer.fit_transform(X) # test init params assert transformer.base == "10" assert transformer.variables == "Age" # test fit attr assert transformer.variables_ == ["Age"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = _to_dict(Xt) + assert result["Age"] == pytest.approx( + [1.30103, 1.32222, 1.27875, 1.25527], abs=1e-5 + ) # test inverse_transform - Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) + Xit = transformer.inverse_transform(Xt) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] def test_error_if_base_value_not_allowed(): - with pytest.raises(ValueError) as record: + msg = "base can take only '10' or 'e' as values. Got other instead." + with pytest.raises(ValueError, match=re.escape(msg)): LogTransformer(base="other") - assert str(record.value) == ( - "base can take only '10' or 'e' as values. Got other instead." - ) -def test_fit_raises_error_if_na_in_df(df_na): - # test case 3: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 4: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) + transformer = LogTransformer() + transformer.fit(X) with pytest.raises(ValueError): - transformer = LogTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_error_if_df_contains_negative_values(df_vartypes): - # test error when data contains negative values - df_neg = df_vartypes.copy() - df_neg.loc[1, "Age"] = -1 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_df_contains_negative_values(make_df): + data_neg = dict(DATA) + data_neg["Age"] = [20, -1, 19, 18] + X = make_df(DATA) + X_neg = make_df(data_neg) - # test case 5: when variable contains negative value, fit + # when variable contains negative value, fit with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_neg) + transformer.fit(X_neg) - # test case 6: when variable contains negative value, transform + # when variable contains negative value, transform with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_neg) + transformer.fit(X) + transformer.transform(X_neg) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) with pytest.raises(NotFittedError): transformer = LogTransformer() - transformer.transform(df_vartypes) + transformer.transform(X) -def test_inverse_e_plus_user_passes_var_list(df_vartypes): - # test case 7: inverse log, user passes variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_e_plus_user_passes_var_list(make_df): + X = make_df(DATA) transformer = LogTransformer(variables="Age") - Xt = transformer.fit_transform(df_vartypes) - X = transformer.inverse_transform(Xt) - - # convert floats to int - X["Age"] = X["Age"].round().astype("int64") + Xt = transformer.fit_transform(X) + Xit = transformer.inverse_transform(Xt) # test init params assert transformer.base == "e" assert transformer.variables == "Age" # test fit attr assert transformer.variables_ == ["Age"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 # test transform output - pd.testing.assert_frame_equal(X, df_vartypes) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] -def test_default_C_preserves_original_fail_fast_behavior(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_default_C_preserves_original_fail_fast_behavior(make_df): """LogTransformer()'s default C=0 must raise at fit() time, with the original exact message, matching pre-merge behavior. See #957.""" - df = pd.DataFrame({"x": [1, 2, 0, 4]}) + X = make_df({"x": [1, 2, 0, 4]}) tr = LogTransformer() assert tr.C == 0 - with pytest.raises(ValueError) as record: - tr.fit(df) - - assert str(record.value) == ( - "Some variables contain zero or negative values, can't apply log" - ) - - -@pytest.fixture(scope="module") -def df_c(): - df = pd.DataFrame( - { - "vara": [0, 1, 2, 3], - "varb": [5, 5, 6, 7], - "varc": [-2, -1, 0, 4], - "vard": [-3, -2, -1, -5], - "vare": ["a", "b", "c", "d"], - } - ) - return df + msg = "Some variables contain zero or negative values, can't apply log" + with pytest.raises(ValueError, match=re.escape(msg)): + tr.fit(X) @pytest.mark.parametrize("c", [1, 0.1, {"var1": 1, "var2": 2}, "auto"]) @@ -181,86 +201,86 @@ def test_c_parameter(c): @pytest.mark.parametrize("c", ["string", [1, 2]]) def test_c_raises_error(c): msg = f"C can take only 'auto', integers, floats or dictionaries. Got {c} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogTransformer(C=c) - assert str(record.value) == msg -def test_C_when_auto(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_auto(make_df): + X = make_df(DATA_C) tr = LogTransformer(C="auto") - tr.fit(df_c) - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - assert tr.C_ == c + tr.fit(X) + assert tr.C_ == DATA_C_AUTO -def test_C_when_dict(df_c): - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - tr = LogTransformer(C=c) - tr.fit(df_c) - assert tr.C_ == c +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_dict(make_df): + X = make_df(DATA_C) + tr = LogTransformer(C=DATA_C_AUTO) + tr.fit(X) + assert tr.C_ == DATA_C_AUTO -def test_C_when_int(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_int(make_df): + X = make_df(DATA_C) tr = LogTransformer(C=10) - tr.fit(df_c) + tr.fit(X) assert tr.C_ == 10 -def test_raises_error_when_transformed_data_has_negative_values_with_C(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_when_transformed_data_has_negative_values_with_C(make_df): + X = make_df(DATA_C) tr = LogTransformer(C="auto") - tr.fit(df_c) - dft = df_c.copy() - dft["vara"] = dft["vara"] - 2 + tr.fit(X) + + data_shifted = dict(DATA_C) + data_shifted["vara"] = [v - 2 for v in DATA_C["vara"]] + Xt = make_df(data_shifted) + msg = ( "Some variables contain zero or negative values after adding constant C, " "can't apply log." ) - with pytest.raises(ValueError) as record: - tr.transform(dft) - assert str(record.value) == msg + with pytest.raises(ValueError, match=re.escape(msg)): + tr.transform(Xt) -def test_log_base_e_with_C(df_c): - dft = LogTransformer(C="auto").fit_transform(df_c) - exp = np.log( - df_c[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("base", ["e", "10"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_with_C(make_df, base): + X = make_df(DATA_C) - dft = LogTransformer(C=10).fit_transform(df_c) - exp = np.log(df_c[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) + dft = LogTransformer(C="auto", base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(DATA_C_AUTO, base) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA_C["vare"] + dft = LogTransformer(C=10, base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(10, base) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA_C["vare"] -def test_log_base_10_with_C(df_c): - dft = LogTransformer(C="auto", base="10").fit_transform(df_c) - exp = np.log10( - df_c[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) - dft = LogTransformer(C=10, base="10").fit_transform(df_c) - exp = np.log10(df_c[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform_with_C(make_df): + X = make_df(DATA_C) - -def test_inverse_transform_with_C(df_c): tr = LogTransformer(C="auto", base="10") - dft = tr.fit_transform(df_c) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df_c, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(DATA_C[var], abs=0.1) tr = LogTransformer(C=10, base="e") - dft = tr.fit_transform(df_c) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df_c, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(DATA_C[var], abs=0.1) diff --git a/tests/test_transformation/test_logcp_transformer.py b/tests/test_transformation/test_logcp_transformer.py index 753cc43bf..98e154735 100644 --- a/tests/test_transformation/test_logcp_transformer.py +++ b/tests/test_transformation/test_logcp_transformer.py @@ -1,10 +1,50 @@ +import re + +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import LogCpTransformer +DATA = { + "vara": [0, 1, 2, 3], + "varb": [5, 5, 6, 7], + "varc": [-2, -1, 0, 4], + "vard": [-3, -2, -1, -5], + "vare": ["a", "b", "c", "d"], +} +DATA_VARS = ["vara", "varb", "varc", "vard"] +DATA_AUTO_C = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} + +DATA_VARTYPES = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + + +def _to_dict(X): + return nw.from_native(X, eager_only=True).to_dict(as_series=False) + + +def _expected_log(c, base): + fn = np.log if base == "e" else np.log10 + out = {} + for var in DATA_VARS: + c_var = c[var] if isinstance(c, dict) else c + out[var] = [fn(x + c_var) for x in DATA[var]] + return out + @pytest.mark.parametrize("base", ["e", "10"]) def test_base_parameter(base): @@ -15,9 +55,8 @@ def test_base_parameter(base): @pytest.mark.parametrize("base", [False, 1, 10]) def test_base_raises_error(base): msg = f"base can take only '10' or 'e' as values. Got {base} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogCpTransformer(base=base) - assert str(record.value) == msg @pytest.mark.parametrize("c", [1, 0.1, {"var1": 1, "var2": 2}, "auto"]) @@ -29,9 +68,8 @@ def test_c_parameter(c): @pytest.mark.parametrize("c", ["string", [1, 2]]) def test_c_raises_error(c): msg = f"C can take only 'auto', integers, floats or dictionaries. Got {c} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogCpTransformer(C=c) - assert str(record.value) == msg def test_instantiation_raises_future_warning(): @@ -40,119 +78,111 @@ def test_instantiation_raises_future_warning(): "LogTransformer and will be removed in version 2.1.0. " 'Use LogTransformer(C="auto") instead.' ) - with pytest.warns(FutureWarning) as record: + with pytest.warns(FutureWarning, match=re.escape(msg)): LogCpTransformer() - assert str(record[0].message) == msg - - -@pytest.fixture(scope="module") -def df(): - df = pd.DataFrame( - { - "vara": [0, 1, 2, 3], - "varb": [5, 5, 6, 7], - "varc": [-2, -1, 0, 4], - "vard": [-3, -2, -1, -5], - "vare": ["a", "b", "c", "d"], - } - ) - return df -def test_C_when_auto(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_auto(make_df): + X = make_df(DATA) tr = LogCpTransformer(C="auto") - tr.fit(df) - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - assert tr.C_ == c + tr.fit(X) + assert tr.C_ == DATA_AUTO_C -def test_C_when_dict(df): - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - tr = LogCpTransformer(C=c) - tr.fit(df) - assert tr.C_ == c +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_dict(make_df): + X = make_df(DATA) + tr = LogCpTransformer(C=DATA_AUTO_C) + tr.fit(X) + assert tr.C_ == DATA_AUTO_C -def test_C_when_int(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_int(make_df): + X = make_df(DATA) tr = LogCpTransformer(C=10) - tr.fit(df) + tr.fit(X) assert tr.C_ == 10 -def test_raises_error_when_transformed_data_has_negative_values(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_when_transformed_data_has_negative_values(make_df): + X = make_df(DATA) tr = LogCpTransformer(C="auto") - tr.fit(df) - dft = df.copy() - dft["vara"] = dft["vara"] - 2 + tr.fit(X) + + data_shifted = dict(DATA) + data_shifted["vara"] = [v - 2 for v in DATA["vara"]] + Xt = make_df(data_shifted) + msg = ( "Some variables contain zero or negative values after adding constant C, " "can't apply log." ) - with pytest.raises(ValueError) as record: - tr.transform(dft) - assert str(record.value) == msg + with pytest.raises(ValueError, match=re.escape(msg)): + tr.transform(Xt) -def test_log_base_e(df): - dft = LogCpTransformer(C="auto").fit_transform(df) - exp = np.log( - df[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) - - dft = LogCpTransformer(C=10).fit_transform(df) - exp = np.log(df[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("base", ["e", "10"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_with_C(make_df, base): + X = make_df(DATA) + dft = LogCpTransformer(C="auto", base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(DATA_AUTO_C, base) + for var in DATA_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA["vare"] -def test_log_base_10(df): - dft = LogCpTransformer(C="auto", base="10").fit_transform(df) - exp = np.log10( - df[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) + dft = LogCpTransformer(C=10, base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(10, base) + for var in DATA_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA["vare"] - dft = LogCpTransformer(C=10, base="10").fit_transform(df) - exp = np.log10(df[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform(make_df): + X = make_df(DATA) -def test_inverse_transform(df): tr = LogCpTransformer(C="auto", base="10") - dft = tr.fit_transform(df) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_VARS: + assert result[var] == pytest.approx(DATA[var], abs=0.1) tr = LogCpTransformer(C=10, base="e") - dft = tr.fit_transform(df) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_VARS: + assert result[var] == pytest.approx(DATA[var], abs=0.1) + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_if_na_in_df(make_df): + X_na = make_df(DATA_NA) + X = make_df(DATA_VARTYPES) -def test_raises_error_if_na_in_df(df_na, df_vartypes): # when dataset contains na, fit method transformer = LogCpTransformer() with pytest.raises(ValueError): - transformer.fit(df_na) + transformer.fit(X_na) # when dataset contains na, transform method transformer = LogCpTransformer() - transformer.fit(df_vartypes) + transformer.fit(X) with pytest.raises(ValueError): - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA_VARTYPES) transformer = LogCpTransformer() with pytest.raises(NotFittedError): - transformer.transform(df_vartypes) + transformer.transform(X) diff --git a/tests/test_transformation/test_power_transformer.py b/tests/test_transformation/test_power_transformer.py index 4f39d8eb0..09fb3eb44 100644 --- a/tests/test_transformation/test_power_transformer.py +++ b/tests/test_transformation/test_power_transformer.py @@ -1,38 +1,57 @@ +import narwhals as nw +import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import PowerTransformer +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + +_exp_ls = [0.001, 0.1, 2, 3, 4, 10] -def test_defo_params_plus_automatically_find_variables(df_vartypes): - # test case 1: automatically select variables - transformer = PowerTransformer(variables=None) - X = transformer.fit_transform(df_vartypes) - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [4.47214, 4.58258, 4.3589, 4.24264] - transf_df["Marks"] = [0.948683, 0.894427, 0.83666, 0.774597] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_defo_params_plus_automatically_find_variables(make_df): + X = make_df(DATA) + transformer = PowerTransformer(variables=None) + Xt = transformer.fit_transform(X) # test init params assert transformer.exp == 0.5 assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) + assert result["Age"] == pytest.approx( + [4.47214, 4.58258, 4.3589, 4.24264], abs=1e-5 + ) + assert result["Marks"] == pytest.approx( + [0.948683, 0.894427, 0.83666, 0.774597], abs=1e-5 + ) # inverse transform - Xit = transformer.inverse_transform(X) + Xit = transformer.inverse_transform(Xt) + result_it = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - Xit["Marks"] = Xit["Marks"].round(1) - - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] def test_error_if_exp_value_not_allowed(): @@ -40,45 +59,48 @@ def test_error_if_exp_value_not_allowed(): PowerTransformer(exp="other") -def test_fit_raises_error_if_na_in_df(df_na): - # test case 2: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = PowerTransformer() - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 3: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) with pytest.raises(ValueError): transformer = PowerTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.fit(X) + transformer.transform(X_na) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) with pytest.raises(NotFittedError): transformer = PowerTransformer() - transformer.transform(df_vartypes) - - -_exp_ls = [0.001, 0.1, 2, 3, 4, 10] + transformer.transform(X) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("exp_base", _exp_ls) -def test_inverse_transform_exp_no_default(exp_base, df_vartypes): +def test_inverse_transform_exp_no_default(make_df, exp_base): + X = make_df(DATA) transformer = PowerTransformer(exp=exp_base) - Xt = transformer.fit_transform(df_vartypes) - X = transformer.inverse_transform(Xt) + Xt = transformer.fit_transform(X) + Xit = transformer.inverse_transform(Xt) + + result_it = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) # convert numbers to original format. - X["Age"] = X["Age"].round().astype("int64") - X["Marks"] = X["Marks"].round(1) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] # test init params - # assert transformer.exp == 100 assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 - # test transform output - pd.testing.assert_frame_equal(X, df_vartypes) + assert transformer.n_features_in_ == 4 diff --git a/tests/test_transformation/test_reciprocal_transformer.py b/tests/test_transformation/test_reciprocal_transformer.py index a8ac99aff..c149e18e8 100644 --- a/tests/test_transformation/test_reciprocal_transformer.py +++ b/tests/test_transformation/test_reciprocal_transformer.py @@ -1,72 +1,94 @@ +import narwhals as nw +import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import ReciprocalTransformer - -def test_automatically_find_variables(df_vartypes): - # test case 1: automatically select variables +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_find_variables_and_inverse_transform(make_df): + X = make_df(DATA) transformer = ReciprocalTransformer(variables=None) - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [0.05, 0.047619, 0.0526316, 0.0555556] - transf_df["Marks"] = [1.11111, 1.25, 1.42857, 1.66667] + Xt = transformer.fit_transform(X) # test init params assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) + assert result["Age"] == pytest.approx( + [0.05, 0.047619, 0.052632, 0.055556], abs=1e-5 + ) + assert result["Marks"] == pytest.approx( + [1.111111, 1.25, 1.428571, 1.666667], abs=1e-5 + ) # test inverse_transform - Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - Xit["Marks"] = Xit["Marks"].round(1) + Xit = transformer.inverse_transform(Xt) + result_it = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) - -def test_fit_raises_error_if_na_in_df(df_na): - # test case 2: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = ReciprocalTransformer() - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 3: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) + transformer = ReciprocalTransformer() + transformer.fit(X) with pytest.raises(ValueError): - transformer = ReciprocalTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_error_if_df_contains_0_as_value(df_vartypes): - # test error when data contains value zero - df_neg = df_vartypes.copy() - df_neg.loc[1, "Age"] = 0 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_df_contains_0_as_value(make_df): + data_zero = dict(DATA) + data_zero["Age"] = [20, 0, 19, 18] + X = make_df(DATA) + X_zero = make_df(data_zero) - # test case 4: when variable contains zero, fit + # when variable contains zero, fit with pytest.raises(ValueError): transformer = ReciprocalTransformer() - transformer.fit(df_neg) + transformer.fit(X_zero) - # test case 5: when variable contains zero, transform + # when variable contains zero, transform + transformer = ReciprocalTransformer() + transformer.fit(X) with pytest.raises(ValueError): - transformer = ReciprocalTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_neg) + transformer.transform(X_zero) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) with pytest.raises(NotFittedError): transformer = ReciprocalTransformer() - transformer.transform(df_vartypes) + transformer.transform(X) diff --git a/tests/test_transformation/test_yeojohnson_transformer.py b/tests/test_transformation/test_yeojohnson_transformer.py index f4eb32f93..b411bddcf 100644 --- a/tests/test_transformation/test_yeojohnson_transformer.py +++ b/tests/test_transformation/test_yeojohnson_transformer.py @@ -1,170 +1,194 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import YeoJohnsonTransformer - -def test_automatically_select_variables(df_vartypes): - # test case 1: automatically select variables +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_automatically_select_variables_and_inverse_transform(make_df): + X = make_df(DATA) transformer = YeoJohnsonTransformer(variables=None) - X = transformer.fit_transform(df_vartypes) - - # expected result - transf_df = df_vartypes.copy() - transf_df["Age"] = [10.167, 10.5406, 9.78774, 9.40229] - transf_df["Marks"] = [0.804449, 0.722367, 0.638807, 0.553652] + Xt = transformer.fit_transform(X) # test init params assert transformer.variables is None - # test fit attr + # test fit attrs assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) + assert result["Age"] == pytest.approx( + [10.167048, 10.540602, 9.787738, 9.402289], abs=1e-5 + ) + assert result["Marks"] == pytest.approx( + [0.804449, 0.722367, 0.638807, 0.553652], abs=1e-5 + ) + + # test inverse_transform, including non-transformed columns + Xit = transformer.inverse_transform(Xt) + result_it = nw.from_native(Xit, eager_only=True).to_dict(as_series=False) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] + assert result_it["Name"] == DATA["Name"] + assert result_it["City"] == DATA["City"] -def test_transformer_on_integer_variables(): - df = pd.DataFrame( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transformer_on_integer_variables(make_df): + X = make_df( { "var1": [0, 1, 0, 2, 3, 4, 5, 6, 8, 10], "var2": [12, 11, 10, 15, 13, 12, 11, 10, 10, 20], } ) - dft = pd.DataFrame( - { - "var1": { - 0: 0.0, - 1: 0.7871467037957388, - 2: 0.0, - 3: 1.34716625120788, - 4: 1.797027857352365, - 5: 2.1794549065159363, - 6: 2.5155129679774246, - 7: 2.817344570368886, - 8: 3.346739213848269, - 9: 3.8051709334268566, - }, - "var2": { - 0: 0.2891005444159968, - 1: 0.2890875957028113, - 2: 0.2890687942494933, - 3: 0.2891213447054929, - 4: 0.2891097235906253, - 5: 0.2891005444159968, - 6: 0.2890875957028113, - 7: 0.2890687942494933, - 8: 0.2890687942494933, - 9: 0.28913341330818815, - }, - } + Xt = YeoJohnsonTransformer().fit_transform(X) + result = nw.from_native(Xt, eager_only=True).to_dict(as_series=False) + + assert result["var1"] == pytest.approx( + [ + 0.0, + 0.787147, + 0.0, + 1.347166, + 1.797028, + 2.179455, + 2.515513, + 2.817345, + 3.346739, + 3.805171, + ], + abs=1e-5, + ) + assert result["var2"] == pytest.approx( + [ + 0.289101, + 0.289088, + 0.289069, + 0.289121, + 0.289110, + 0.289101, + 0.289088, + 0.289069, + 0.289069, + 0.289133, + ], + abs=1e-5, ) - - X_tr = YeoJohnsonTransformer().fit_transform(df) - pd.testing.assert_frame_equal(X_tr, dft) -def test_fit_raises_error_if_na_in_df(df_na): - # test case 2: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = YeoJohnsonTransformer() - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 3: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) + transformer = YeoJohnsonTransformer() + transformer.fit(X) with pytest.raises(ValueError): - transformer = YeoJohnsonTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) with pytest.raises(NotFittedError): transformer = YeoJohnsonTransformer() - transformer.transform(df_vartypes) - - -def test_inverse_transform_automatically_select_only_transformed_columns(df_vartypes): - X = df_vartypes.copy(deep=True) - transformer = YeoJohnsonTransformer(variables=None) - X_trans = transformer.fit_transform(X) + transformer.transform(X) - X_inverse = transformer.inverse_transform(X_trans) - X_inverse["Age"] = X_inverse["Age"].round(0).astype(int) - pd.testing.assert_frame_equal(X, X_inverse, check_dtype=False) - - -def test_inverse_with_X_negative_and_positive(): - X = pd.DataFrame( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_with_x_negative_and_positive(make_df): + X = make_df( { - "var1": np.arange(-20, 0), - "var2": np.arange(0, 20), - "var3": np.arange(-10, 10), + "var1": list(np.arange(-20, 0)), + "var2": list(np.arange(0, 20)), + "var3": list(np.arange(-10, 10)), } ) transformer = YeoJohnsonTransformer(variables=None) - X_trans = transformer.fit_transform(X) - - X_inverse = transformer.inverse_transform(X_trans) - X_inverse = X_inverse.round(0).astype(int) + Xt = transformer.fit_transform(X) + Xi = transformer.inverse_transform(Xt) + result = nw.from_native(Xi, eager_only=True).to_dict(as_series=False) - pd.testing.assert_frame_equal(X, X_inverse, check_dtype=False) + assert [round(v) for v in result["var1"]] == list(np.arange(-20, 0)) + assert [round(v) for v in result["var2"]] == list(np.arange(0, 20)) + assert [round(v) for v in result["var3"]] == list(np.arange(-10, 10)) -def test_inverse_with_with_non_linear_index(): +def test_inverse_with_non_linear_index(): + # pandas-specific: exercises index-preserving behaviour, which has no + # polars equivalent (polars has no row index). X = pd.DataFrame( { "var1": np.arange(-20, 0), "var2": np.arange(0, 20), "var3": np.arange(-10, 10), }, - index=[13, 15, 12, 11, 17, 9, 4, 0, 1, 14, 18, 2, 3, 6, 5, 7, 8, 2, 16, 10] + index=[13, 15, 12, 11, 17, 9, 4, 0, 1, 14, 18, 2, 3, 6, 5, 7, 8, 2, 16, 10], ) transformer = YeoJohnsonTransformer(variables=None) - X_trans = transformer.fit_transform(X) + Xt = transformer.fit_transform(X) - X_inverse = transformer.inverse_transform(X_trans) - X_inverse = X_inverse.round(0).astype(int) + Xi = transformer.inverse_transform(Xt) + Xi = Xi.round(0).astype(int) - pd.testing.assert_frame_equal(X, X_inverse, check_dtype=False) + pd.testing.assert_frame_equal(X, Xi, check_dtype=False) -def test_lambda_equals_lambda_equal_0(): - X = pd.DataFrame( - { - "var1": np.arange(0, 20), - "var2": np.arange(20, 40), - } - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_lambda_equal_0(make_df): + X = make_df({"var1": list(np.arange(0, 20)), "var2": list(np.arange(20, 40))}) transformer = YeoJohnsonTransformer(variables=None) transformer = transformer.fit(X) - transformer.lambda_dict_ = {"var1": 0, "var2": 0} - X_trans = transformer.transform(X) - X_inverse = transformer.inverse_transform(X_trans) - X_inverse = X_inverse.round(0).astype(int) + Xt = transformer.transform(X) + Xi = transformer.inverse_transform(Xt) + result = nw.from_native(Xi, eager_only=True).to_dict(as_series=False) - pd.testing.assert_frame_equal(X, X_inverse, check_dtype=False) + assert [round(v) for v in result["var1"]] == list(np.arange(0, 20)) + assert [round(v) for v in result["var2"]] == list(np.arange(20, 40)) -def test_lambda_equals_lambda_equal_2(): - X = pd.DataFrame({"var1": np.arange(-21, -1), "var2": np.arange(-41, -21)}) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_lambda_equal_2(make_df): + X = make_df({"var1": list(np.arange(-21, -1)), "var2": list(np.arange(-41, -21))}) transformer = YeoJohnsonTransformer(variables=None) transformer = transformer.fit(X) - transformer.lambda_dict_ = {"var1": 2, "var2": 2} - X_trans = transformer.transform(X) - X_inverse = transformer.inverse_transform(X_trans) - X_inverse = X_inverse.round(0).astype(int) + Xt = transformer.transform(X) + Xi = transformer.inverse_transform(Xt) + result = nw.from_native(Xi, eager_only=True).to_dict(as_series=False) - pd.testing.assert_frame_equal(X, X_inverse, check_dtype=False) + assert [round(v) for v in result["var1"]] == list(np.arange(-21, -1)) + assert [round(v) for v in result["var2"]] == list(np.arange(-41, -21)) diff --git a/tests/test_variable_handling/conftest.py b/tests/test_variable_handling/conftest.py index 841656da2..536776c93 100644 --- a/tests/test_variable_handling/conftest.py +++ b/tests/test_variable_handling/conftest.py @@ -1,7 +1,40 @@ +from datetime import datetime, timezone + import pandas as pd +import polars as pl import pytest +def cast_categorical(df, columns): + """Cast `columns` to the backend's categorical dtype, whichever backend `df` + (pandas or polars) happens to be. Used to build matched pandas/polars data + for tests parametrized over both libraries. + """ + if isinstance(df, pd.DataFrame): + df = df.copy() + df[columns] = df[columns].astype("category") + return df + return df.with_columns([pl.col(c).cast(pl.Categorical) for c in columns]) + + +# Data shared between the pandas and polars variants of a test. +BASIC_DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + +DATETIME_DATA = { + **BASIC_DATA, + "date_range": [datetime(2020, 2, 24, 0, i) for i in range(4)], + "date_obj0": ["2020-02-24", "2020-02-25", "2020-02-26", "2020-02-27"], + "date_range_tz": [ + datetime(2020, 2, 24, 0, i, tzinfo=timezone.utc) for i in range(4) + ], +} + + @pytest.fixture def df(): df = pd.DataFrame( diff --git a/tests/test_variable_handling/test_check_variables.py b/tests/test_variable_handling/test_check_variables.py index 8eba88cb0..eb1bb018a 100644 --- a/tests/test_variable_handling/test_check_variables.py +++ b/tests/test_variable_handling/test_check_variables.py @@ -1,4 +1,5 @@ import pandas as pd +import polars as pl import pytest from feature_engine.variable_handling import ( @@ -7,103 +8,139 @@ check_datetime_variables, check_numerical_variables, ) +from tests.test_variable_handling.conftest import ( + BASIC_DATA, + DATETIME_DATA, + cast_categorical, +) -def test_check_numerical_variables_returns_numerical_variables(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_numerical_variables_returns_numerical_variables(make_df): + df = make_df(BASIC_DATA) assert check_numerical_variables(df, ["Age", "Marks"]) == ["Age", "Marks"] assert check_numerical_variables(df, ["Age"]) == ["Age"] assert check_numerical_variables(df, "Age") == ["Age"] + + +def test_check_numerical_variables_returns_numerical_variables_int_names(df_int): + # polars requires string column names, so int-named columns are pandas-only assert check_numerical_variables(df_int, [3, 4]) == [3, 4] assert check_numerical_variables(df_int, [3]) == [3] assert check_numerical_variables(df_int, 4) == [4] -def test_check_numerical_variables_raises_errors_when_not_numerical(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_numerical_variables_raises_errors_when_not_numerical(make_df): + df = make_df(BASIC_DATA) msg = ( "Some of the variables are not numerical. Please cast them as " "numerical before using this transformer." ) - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df, "Name") - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df, "Name") + + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df, ["Name"]) - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df, ["Name"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df, ["Name", "Marks"]) - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df_int, 1) - assert str(record.value) == msg - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df_int, [1]) - assert str(record.value) == msg +def test_check_numerical_variables_raises_errors_int_names(df_int): + msg = ( + "Some of the variables are not numerical. Please cast them as " + "numerical before using this transformer." + ) + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df_int, 1) - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df, ["Name", "Marks"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df_int, [1]) - with pytest.raises(TypeError) as record: - assert check_numerical_variables(df_int, [2, 3]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_numerical_variables(df_int, [2, 3]) -def test_check_categorical_variables_returns_categorical_variables(df, df_int): - assert check_categorical_variables(df, ["Name", "date_obj0"]) == [ - "Name", - "date_obj0", - ] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_categorical_variables_returns_categorical_variables(make_df): + df = make_df(BASIC_DATA) + assert check_categorical_variables(df, ["Name", "City"]) == ["Name", "City"] assert check_categorical_variables(df, ["Name"]) == ["Name"] - assert check_categorical_variables(df, "date_obj0") == ["date_obj0"] + assert check_categorical_variables(df, "Name") == ["Name"] + + +def test_check_categorical_variables_numeric_categories_pandas_only(): + # polars categoricals are always string-backed, so casting a + # numeric column to Categorical isn't a realistic polars scenario. + df = pd.DataFrame(BASIC_DATA) + df = cast_categorical(df, ["Age", "Marks"]) + assert check_categorical_variables(df, ["Age", "Marks"]) == ["Age", "Marks"] + + +def test_check_categorical_variables_returns_categorical_variables_int_names(df_int): assert check_categorical_variables(df_int, [1, 2]) == [1, 2] assert check_categorical_variables(df_int, [2]) == [2] assert check_categorical_variables(df_int, 2) == [2] - df[["Age", "Marks"]] = df[["Age", "Marks"]].astype(pd.CategoricalDtype) - assert check_categorical_variables(df, ["Age", "Marks"]) == ["Age", "Marks"] - -def test_check_categorical_variables_raises_errors_when_not_categorical(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_categorical_variables_raises_errors_when_not_categorical(make_df): + df = make_df(BASIC_DATA) msg = ( "Some of the variables are not categorical. Please cast them as " "object or categorical before using this transformer." ) - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df, "Age") - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df, "Age") + + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df, ["Age"]) - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df, ["Age"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df, ["Name", "Marks"]) - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df_int, 3) - assert str(record.value) == msg - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df_int, [3]) - assert str(record.value) == msg +def test_check_categorical_variables_raises_errors_int_names(df_int): + msg = ( + "Some of the variables are not categorical. Please cast them as " + "object or categorical before using this transformer." + ) + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df_int, 3) - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df, ["Name", "Marks"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df_int, [3]) - with pytest.raises(TypeError) as record: - assert check_categorical_variables(df_int, [2, 3]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_categorical_variables(df_int, [2, 3]) -def test_check_datetime_variables_returns_datetime_variables(df_datetime): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_datetime_variables_returns_datetime_variables(make_df): + df = make_df(DATETIME_DATA) var_dt = ["date_range"] var_dt_str = "date_range" + vars_dt = ["date_range", "date_obj0", "date_range_tz"] + tz_time = "date_range_tz" + + assert check_datetime_variables(df, var_dt_str) == [var_dt_str] + assert check_datetime_variables(df, var_dt) == var_dt + assert check_datetime_variables(df, vars_dt) == vars_dt + assert check_datetime_variables(df, tz_time) == [tz_time] + + # only the string column can be cast to categorical. Native Datetime + # columns can't be cast to Categorical in polars + df = cast_categorical(df, ["date_obj0"]) + assert check_datetime_variables(df, "date_obj0") == ["date_obj0"] + + +def test_check_datetime_variables_returns_pandas_only_string_formats(df_datetime): + # "01-Jan-2010"-style and "10/11/12"-style strings are recognised via + # flexible, dateutil-backed guessing. vars_convertible_to_dt = ["date_range", "date_obj1", "date_obj2", "time_obj"] var_convertible_to_dt = "date_obj1" - tz_time = "time_objTZ" - tz_time_obj = "date_range_tz" - # when variables are specified - assert check_datetime_variables(df_datetime, var_dt_str) == [var_dt_str] - assert check_datetime_variables(df_datetime, var_dt) == var_dt assert check_datetime_variables(df_datetime, var_convertible_to_dt) == [ var_convertible_to_dt ] @@ -111,8 +148,6 @@ def test_check_datetime_variables_returns_datetime_variables(df_datetime): check_datetime_variables(df_datetime, vars_convertible_to_dt) == vars_convertible_to_dt ) - assert check_datetime_variables(df_datetime, tz_time) == [tz_time] - assert check_datetime_variables(df_datetime, tz_time_obj) == [tz_time_obj] df_datetime[vars_convertible_to_dt] = df_datetime[vars_convertible_to_dt].astype( pd.CategoricalDtype @@ -123,55 +158,48 @@ def test_check_datetime_variables_returns_datetime_variables(df_datetime): ) -def test_check_datetime_variables_raises_errors_when_not_datetime(df_datetime): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_datetime_variables_raises_errors_when_not_datetime(make_df): + df = make_df(DATETIME_DATA) msg = "Some of the variables are not or cannot be parsed as datetime." - with pytest.raises(TypeError) as record: - assert check_datetime_variables(df_datetime, variables="Age") - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_datetime_variables(df, variables="Age") - with pytest.raises(TypeError) as record: - assert check_datetime_variables(df_datetime, variables=["Age", "Name"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_datetime_variables(df, variables=["Age", "Name"]) - with pytest.raises(TypeError): - assert check_datetime_variables(df_datetime, variables=["date_range", "Age"]) - assert str(record.value) == msg + with pytest.raises(TypeError, match=msg): + check_datetime_variables(df, variables=["date_range", "Age"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_vars", [ - ["Name", "City", "Age", "Marks", "dob"], - [ - "Name", - "City", - "Age", - "Marks", - ], + ["Name", "City", "Age", "Marks"], + ["Name", "City", "Age"], "Name", ["Age"], ], ) -def test_check_all_variables_returns_all_variables(df_vartypes, input_vars): +def test_check_all_variables_returns_all_variables(make_df, input_vars): + df = make_df(BASIC_DATA) if isinstance(input_vars, list): - assert check_all_variables(df_vartypes, input_vars) == input_vars + assert check_all_variables(df, input_vars) == input_vars else: - assert check_all_variables(df_vartypes, input_vars) == [input_vars] + assert check_all_variables(df, input_vars) == [input_vars] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "input_vars", [["Name", "City", "Absent"], "Absent", ["Absent"]] ) -def test_check_all_variables_raises_errors_when_not_in_dataframe( - df_vartypes, input_vars -): +def test_check_all_variables_raises_errors_when_not_in_dataframe(make_df, input_vars): + df = make_df(BASIC_DATA) msg_ls = "'Some of the variables are not in the dataframe.'" msg_single = "'The variable Absent is not in the dataframe.'" + msg = msg_ls if isinstance(input_vars, list) else msg_single - with pytest.raises(KeyError) as record: - assert check_all_variables(df_vartypes, input_vars) - if isinstance(input_vars, list): - assert str(record.value) == msg_ls - else: - assert str(record.value) == msg_single + with pytest.raises(KeyError, match=msg): + check_all_variables(df, input_vars) diff --git a/tests/test_variable_handling/test_fe_type_checks.py b/tests/test_variable_handling/test_fe_type_checks.py deleted file mode 100644 index de4bc2d38..000000000 --- a/tests/test_variable_handling/test_fe_type_checks.py +++ /dev/null @@ -1,93 +0,0 @@ -import pandas as pd - -from feature_engine.variable_handling._variable_type_checks import ( - _is_categorical_and_is_datetime, - _is_categorical_and_is_not_datetime, - _is_categories_num, - _is_convertible_to_dt, - _is_convertible_to_num, -) - - -def test_is_categories_num(df): - assert _is_categories_num(df["Name"]) is False - - df["Age"] = df["Age"].astype("category") - assert _is_categories_num(df["Age"]) is True - - -def test_is_convertible_to_num(df): - assert _is_convertible_to_num(df["Name"]) is False - assert _is_convertible_to_num(df["date_obj0"]) is False - - df["age_str"] = ["20", "21", "19", "18"] - assert _is_convertible_to_num(df["age_str"]) is True - - -def test_is_convertible_to_dt(df): - assert _is_convertible_to_dt(df["date_obj0"]) is True - assert _is_convertible_to_dt(df["date_range"]) is True - assert _is_convertible_to_dt(df["Name"]) is False - - df["age_str"] = ["20", "21", "19", "18"] - assert _is_convertible_to_dt(df["age_str"]) is False - - -def test_is_categorical_and_is_datetime(df, df_datetime): - assert _is_categorical_and_is_datetime(df["date_obj0"]) is True - assert _is_categorical_and_is_datetime(df["Name"]) is False - assert _is_categorical_and_is_datetime(df_datetime["date_obj1"]) is True - - df["age_str"] = ["20", "21", "19", "18"] - assert _is_categorical_and_is_datetime(df["age_str"]) is False - - df = df.copy() - # from pandas 3 onwards, object types that contain strings are not recognised as - # objects any more - df["Age"] = df["Age"].astype("O") - assert _is_categorical_and_is_datetime(df["Age"]) is False - - # Object Datetime - s_obj_dt = pd.Series([pd.Timestamp("2020-01-01")], dtype="object") - assert _is_categorical_and_is_datetime(s_obj_dt) is True - - # StringDtype Datetime (if convertible) - s_str_dt = pd.Series(["2020-01-01", "2020-01-02"], dtype="string") - assert _is_categorical_and_is_datetime(s_str_dt) is True - - # Numeric (should be False for both if and elif branches) - s_num = pd.Series([1, 2, 3]) - assert _is_categorical_and_is_datetime(s_num) is False - - # Categorical (should hit the 'if' branch) - s_cat = pd.Series(["a", "b"], dtype="category") - assert _is_categorical_and_is_datetime(s_cat) is False - - -def test_is_categorical_and_is_not_datetime(df): - assert _is_categorical_and_is_not_datetime(df["date_obj0"]) is False - assert _is_categorical_and_is_not_datetime(df["date_obj0"]) is False - assert _is_categorical_and_is_not_datetime(df["Name"]) is True - - df["age_str"] = ["20", "21", "19", "18"] - assert _is_categorical_and_is_not_datetime(df["age_str"]) is True - - # Object Integer - s_obj_int = pd.Series([1, 2], dtype="object") - assert _is_categorical_and_is_not_datetime(s_obj_int) is True - - # Object Datetime should be False - s_obj_dt = pd.Series([pd.Timestamp("2020-01-01")], dtype="object") - assert _is_categorical_and_is_not_datetime(s_obj_dt) is False - - # StringDtype (not convertible to numeric/datetime) should be True - s_str = pd.Series(["a", "b"], dtype="string") - assert _is_categorical_and_is_not_datetime(s_str) is True - - # Numeric should be False - s_num = pd.Series([1, 2, 3]) - assert _is_categorical_and_is_not_datetime(s_num) is False - - # Categorical should be True (it hits the 'if' branch) - s_cat = pd.Series(["a", "b"], dtype="category") - assert _is_categorical_and_is_not_datetime(s_cat) is True diff --git a/tests/test_variable_handling/test_find_variables.py b/tests/test_variable_handling/test_find_variables.py index 6ae29384d..249eb0b37 100644 --- a/tests/test_variable_handling/test_find_variables.py +++ b/tests/test_variable_handling/test_find_variables.py @@ -1,4 +1,5 @@ import pandas as pd +import polars as pl import pytest from feature_engine.variable_handling import ( @@ -8,89 +9,100 @@ find_datetime_variables, find_numerical_variables, ) +from tests.test_variable_handling.conftest import ( + BASIC_DATA, + DATETIME_DATA, + cast_categorical, +) # --- find_numerical_variables --- # -def test_numerical_variables_finds_variables(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numerical_variables_finds_variables(make_df): + df = make_df(BASIC_DATA) assert find_numerical_variables(df) == ["Age", "Marks"] + + +def test_numerical_variables_finds_variables_with_int_column_names(df_int): + # polars requires string column names. int-named columns are pandas-only assert find_numerical_variables(df_int) == [3, 4] -def test_numerical_variables_raises_error(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numerical_variables_raises_error(make_df): + df = make_df(BASIC_DATA) msg = "No numerical variables found in this dataframe." with pytest.raises(TypeError, match=msg): - find_numerical_variables(df.drop(["Age", "Marks"], axis=1)) - - with pytest.raises(TypeError, match=msg): - find_numerical_variables(df_int.drop([3, 4], axis=1)) + find_numerical_variables(df[["Name", "City"]]) -def test_numerical_variables_raises_warning(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numerical_variables_raises_warning(make_df): + df = make_df(BASIC_DATA) msg = "No numerical variables found in this dataframe." - - # Test with a regular DataFrame with pytest.warns(UserWarning, match=msg): - find_numerical_variables(df.drop(["Age", "Marks"], axis=1), return_empty=True) - - # Test with integer-only DataFrame - with pytest.warns(UserWarning, match=msg): - find_numerical_variables(df_int.drop([3, 4], axis=1), return_empty=True) + find_numerical_variables(df[["Name", "City"]], return_empty=True) -def test_numerical_variables_returns_empty_list(df, df_int): - assert ( - find_numerical_variables(df.drop(["Age", "Marks"], axis=1), return_empty=True) - == [] - ) - assert ( - find_numerical_variables(df_int.drop([3, 4], axis=1), return_empty=True) == [] - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numerical_variables_returns_empty_list(make_df): + df = make_df(BASIC_DATA) + assert find_numerical_variables(df[["Name", "City"]], return_empty=True) == [] # --- find_categorical_variables --- # -def test_categorical_variables_finds_variables(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_categorical_variables_finds_variables(make_df): + df = make_df(BASIC_DATA) assert find_categorical_variables(df) == ["Name", "City"] + + +def test_categorical_variables_finds_variables_with_int_column_names(df_int): assert find_categorical_variables(df_int) == [1, 2] -def test_categorical_variables_raises_error(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_categorical_variables_raises_error(make_df): + df = make_df(BASIC_DATA) msg = "No categorical variables found in this dataframe." with pytest.raises(TypeError, match=msg): - find_categorical_variables(df.drop(["Name", "City"], axis=1)) - - with pytest.raises(TypeError, match=msg): - find_categorical_variables(df_int.drop([1, 2], axis=1)) + find_categorical_variables(df[["Age", "Marks"]]) -def test_categorical_variables_raises_warning(df, df_int): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_categorical_variables_raises_warning(make_df): + df = make_df(BASIC_DATA) msg = "No categorical variables found in this dataframe." - - # Test with a regular DataFrame with pytest.warns(UserWarning, match=msg): - find_categorical_variables(df.drop(["Name", "City"], axis=1), return_empty=True) - - # Test with integer-only DataFrame - with pytest.warns(UserWarning, match=msg): - find_categorical_variables(df_int.drop([1, 2], axis=1), return_empty=True) + find_categorical_variables(df[["Age", "Marks"]], return_empty=True) -def test_categorical_variables_returns_empty_list(df, df_int): - assert ( - find_categorical_variables(df.drop(["Name", "City"], axis=1), return_empty=True) - == [] - ) - assert ( - find_categorical_variables(df_int.drop([1, 2], axis=1), return_empty=True) == [] - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_categorical_variables_returns_empty_list(make_df): + df = make_df(BASIC_DATA) + assert find_categorical_variables(df[["Age", "Marks"]], return_empty=True) == [] # --- find_datetime_variables --- # -def test_datetime_variables_finds_variables(df_datetime): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_datetime_variables_finds_variables(make_df): + df = make_df(DATETIME_DATA) + vars_dt = ["date_range", "date_obj0", "date_range_tz"] + assert find_datetime_variables(df) == vars_dt + + assert find_datetime_variables( + df[["date_obj0", "date_range", "date_range_tz"]], + ) == ["date_obj0", "date_range", "date_range_tz"] + + +def test_datetime_variables_finds_pandas_only_string_formats(df_datetime): + # "01-Jan-2010"-style, "10/11/12"-style and bare-time strings are + # recognised through flexible, dateutil-backed guessing. vars_dt = [ "date_range", "date_obj0", @@ -100,206 +112,206 @@ def test_datetime_variables_finds_variables(df_datetime): "time_obj", "time_objTZ", ] - assert find_datetime_variables(df_datetime) == vars_dt - assert find_datetime_variables( - df_datetime[vars_dt].reindex(columns=["date_obj1", "date_range", "date_obj2"]), - ) == ["date_obj1", "date_range", "date_obj2"] +def test_datetime_variables_finds_flexible_string_formats_in_polars_too(): + # flexible, dateutil-backed date guessing is backend-agnostic, so polars + # now also recognises non-ISO formats it previously could not. + df = pl.DataFrame( + { + "var_num": [1, 2, 3], + "date_obj1": ["01-Jan-2010", "24-Feb-1945", "14-Jun-2100"], + "date_obj2": ["10/11/12", "12/31/09", "06/30/95"], + } + ) + assert find_datetime_variables(df) == ["date_obj1", "date_obj2"] -def test_datetime_variables_raises_error(df_datetime): - msg = "No datetime variables found in this dataframe." +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_datetime_variables_raises_error(make_df): + df = make_df(DATETIME_DATA) + msg = "No datetime variables found in this dataframe." vars_nondt = ["Marks", "Age", "Name"] - with pytest.raises(TypeError, match=msg): - find_datetime_variables(df_datetime.loc[:, vars_nondt]) + find_datetime_variables(df[vars_nondt]) -def test_datetime_variables_raises_warning(df_datetime): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_datetime_variables_raises_warning(make_df): + df = make_df(DATETIME_DATA) msg = "No datetime variables found in this dataframe." vars_nondt = ["Marks", "Age", "Name"] with pytest.warns(UserWarning, match=msg): - find_datetime_variables(df_datetime.loc[:, vars_nondt], return_empty=True) + find_datetime_variables(df[vars_nondt], return_empty=True) -def test_datetime_variables_returns_empty_list(df_datetime): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_datetime_variables_returns_empty_list(make_df): + df = make_df(DATETIME_DATA) vars_nondt = ["Marks", "Age", "Name"] - assert ( - find_datetime_variables(df_datetime.loc[:, vars_nondt], return_empty=True) == [] - ) + assert find_datetime_variables(df[vars_nondt], return_empty=True) == [] # --- find_all_variables --- # -def test_find_all_variables(df): - all_vars = [ - "Name", - "City", - "Age", - "Marks", - "date_range", - "date_obj0", - "date_range_tz", - ] - assert find_all_variables(df, exclude_datetime=False) == all_vars +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_find_all_variables(make_df): + df = make_df(BASIC_DATA) + assert find_all_variables(df, exclude_datetime=False) == list(BASIC_DATA.keys()) -def test_find_all_variables_excludes_dt(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_find_all_variables_excludes_dt(make_df): + df = make_df(DATETIME_DATA) all_vars_no_dt = ["Name", "City", "Age", "Marks"] assert find_all_variables(df, exclude_datetime=True) == all_vars_no_dt -def test_find_all_variables_raises_error(df): - dt_vars = [ - "date_range", - "date_obj0", - "date_range_tz", - ] - df = df[dt_vars] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_find_all_variables_raises_error(make_df): + dt_vars = ["date_range", "date_obj0", "date_range_tz"] + df = make_df(DATETIME_DATA)[dt_vars] msg = "No variables found in this dataframe" with pytest.raises(TypeError, match=msg): find_all_variables(df, exclude_datetime=True) -def test_find_all_variables_raises_warning(df): - dt_vars = [ - "date_range", - "date_obj0", - "date_range_tz", - ] - df = df[dt_vars] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_find_all_variables_raises_warning(make_df): + dt_vars = ["date_range", "date_obj0", "date_range_tz"] + df = make_df(DATETIME_DATA)[dt_vars] msg = "No variables found in this dataframe" with pytest.warns(UserWarning, match=msg): find_all_variables(df, exclude_datetime=True, return_empty=True) -def test_find_all_variables_returns_empty(df): - dt_vars = [ - "date_range", - "date_obj0", - "date_range_tz", - ] - df = df[dt_vars] +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_find_all_variables_returns_empty(make_df): + dt_vars = ["date_range", "date_obj0", "date_range_tz"] + df = make_df(DATETIME_DATA)[dt_vars] assert find_all_variables(df, exclude_datetime=True, return_empty=True) == [] # --- find_categorical_and_numerical_variables --- # -def test_numcat_user_passes_varlist(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_user_passes_varlist(make_df): + df = make_df(BASIC_DATA) + # Case 1: user passes 1 variable that is categorical - assert find_categorical_and_numerical_variables(df_vartypes, ["Name"]) == ( - ["Name"], - [], - ) - assert find_categorical_and_numerical_variables(df_vartypes, "Name") == ( - ["Name"], - [], - ) + assert find_categorical_and_numerical_variables(df, ["Name"]) == (["Name"], []) + assert find_categorical_and_numerical_variables(df, "Name") == (["Name"], []) # Case 2: user passes 1 variable that is numerical - assert find_categorical_and_numerical_variables(df_vartypes, ["Age"]) == ( - [], - ["Age"], - ) - assert find_categorical_and_numerical_variables(df_vartypes, "Age") == ( - [], - ["Age"], - ) + assert find_categorical_and_numerical_variables(df, ["Age"]) == ([], ["Age"]) + assert find_categorical_and_numerical_variables(df, "Age") == ([], ["Age"]) # Case 3: user passes 1 categorical and 1 numerical variable - assert find_categorical_and_numerical_variables(df_vartypes, ["Age", "Name"]) == ( + assert find_categorical_and_numerical_variables(df, ["Age", "Name"]) == ( ["Name"], ["Age"], ) -def test_numcat_when_var_is_none(df_vartypes): - # Case 4: automatically identify variables - assert find_categorical_and_numerical_variables(df_vartypes, None) == ( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_when_var_is_none(make_df): + df = make_df(BASIC_DATA) + + assert find_categorical_and_numerical_variables(df, None) == ( ["Name", "City"], ["Age", "Marks"], ) - assert find_categorical_and_numerical_variables( - df_vartypes[["Name", "City"]], None - ) == (["Name", "City"], []) - assert find_categorical_and_numerical_variables( - df_vartypes[["Age", "Marks"]], None - ) == ([], ["Age", "Marks"]) - - -@pytest.fixture(scope="module") -def dfdt(): - X = pd.DataFrame() - X["date1"] = pd.date_range("2020-02-24", periods=1000, freq="min") - X["date2"] = pd.date_range("2021-09-29", periods=1000, freq="h") - X["date3"] = ["2020-02-24"] * 1000 - return X + assert find_categorical_and_numerical_variables(df[["Name", "City"]], None) == ( + ["Name", "City"], + [], + ) + assert find_categorical_and_numerical_variables(df[["Age", "Marks"]], None) == ( + [], + ["Age", "Marks"], + ) -def test_numcat_raises_no_var_error(dfdt): +@pytest.mark.parametrize( + "make_df, assert_error", [(pd.DataFrame, TypeError), (pl.DataFrame, TypeError)] +) +def test_numcat_raises_no_var_error(make_df, assert_error): # Case 5: error when no variable is numerical or categorical + df = make_df( + { + "date1": DATETIME_DATA["date_range"], + "date2": DATETIME_DATA["date_range_tz"], + } + ) msg = "There are no numerical or categorical variables" - with pytest.raises(TypeError, match=msg): - find_categorical_and_numerical_variables(dfdt, None) + with pytest.raises(assert_error, match=msg): + find_categorical_and_numerical_variables(df, None) msg = "The variable entered is neither numerical nor categorical." - with pytest.raises(TypeError, match=msg): - find_categorical_and_numerical_variables(dfdt, "date1") + with pytest.raises(assert_error, match=msg): + find_categorical_and_numerical_variables(df, "date1") -def test_numcat_raises_no_var_warn(dfdt): - # Case 6: warning when no variable is numerical or categorical +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_raises_no_var_warn(make_df): + df = make_df( + { + "date1": DATETIME_DATA["date_range"], + "date2": DATETIME_DATA["date_range_tz"], + } + ) msg = "There are no numerical or categorical variables" with pytest.warns(UserWarning, match=msg): - find_categorical_and_numerical_variables( - dfdt, - None, - return_empty=True, - ) + find_categorical_and_numerical_variables(df, None, return_empty=True) msg = "The variable entered is neither numerical nor" with pytest.warns(UserWarning, match=msg): find_categorical_and_numerical_variables( - dfdt, variables="date1", return_empty=True + df, variables="date1", return_empty=True ) -def test_numcat_returns_empty_lists(dfdt): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_returns_empty_lists(make_df): + df = make_df( + { + "date1": DATETIME_DATA["date_range"], + "date2": DATETIME_DATA["date_range_tz"], + } + ) assert find_categorical_and_numerical_variables( - dfdt, - None, - return_empty=True, + df, None, return_empty=True ) == ([], []) assert find_categorical_and_numerical_variables( - dfdt, - "date1", - return_empty=True, + df, "date1", return_empty=True ) == ([], []) -def test_numcat_on_user_empty_list(df_vartypes): - # Case 7: user passes empty list +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_on_user_empty_list(make_df): + df = make_df(BASIC_DATA) + msg = "The list of variables provided is empty. If this was" with pytest.raises(ValueError, match=msg): - find_categorical_and_numerical_variables(df_vartypes, []) + find_categorical_and_numerical_variables(df, []) msg = "The list of variables provided is empty. Returning " with pytest.warns(UserWarning, match=msg): - find_categorical_and_numerical_variables(df_vartypes, [], return_empty=True) + find_categorical_and_numerical_variables(df, [], return_empty=True) - assert find_categorical_and_numerical_variables( - df_vartypes, [], return_empty=True - ) == ([], []) + assert find_categorical_and_numerical_variables(df, [], return_empty=True) == ( + [], + [], + ) def test_numcat_when_dt_as_object(df_vartypes): - # Case 8: datetime cast as object + # Case 8: datetime cast as object - pandas-only, `df_vartypes["dob"]` is a + # pandas datetime64 column relying on pandas' `.astype("O")`, which has no + # polars equivalent (polars has no generic object dtype to cast into). df = df_vartypes.copy() df["dob"] = df["dob"].astype("O") - # datetime variable is skipped when automatically finding variables, assert find_categorical_and_numerical_variables(df, None) == ( ["Name", "City"], ["Age", "Marks"], @@ -310,12 +322,43 @@ def test_numcat_when_dt_as_object(df_vartypes): ) -def test_numcat_vars_as_category(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_vars_as_category(make_df): # Case 9: variables cast as category - df = df_vartypes.copy() - df["City"] = df["City"].astype("category") + df = make_df(BASIC_DATA) + df = cast_categorical(df, ["City"]) assert find_categorical_and_numerical_variables(df, None) == ( ["Name", "City"], ["Age", "Marks"], ) assert find_categorical_and_numerical_variables(df, "City") == (["City"], []) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_agrees_with_find_categorical_on_date_like_category(make_df): + # Regression test: the single-variable path used to disagree with + # find_categorical_variables on a date-like category column. + df = make_df({"date_cat": DATETIME_DATA["date_obj0"], "num": BASIC_DATA["Age"]}) + df = cast_categorical(df, ["date_cat"]) + + assert find_categorical_variables(df, return_empty=True) == [] + assert find_categorical_and_numerical_variables(df, None) == ([], ["num"]) + assert find_categorical_and_numerical_variables( + df, "date_cat", return_empty=True + ) == ([], []) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_numcat_exclude_datetime_false_keeps_date_like_category(make_df): + # exclude_datetime=False must be honoured consistently across all three + # entry points, including the single-variable branch. + df = make_df({"date_cat": DATETIME_DATA["date_obj0"], "num": BASIC_DATA["Age"]}) + df = cast_categorical(df, ["date_cat"]) + + assert find_categorical_variables(df, exclude_datetime=False) == ["date_cat"] + assert find_categorical_and_numerical_variables( + df, None, exclude_datetime=False + ) == (["date_cat"], ["num"]) + assert find_categorical_and_numerical_variables( + df, "date_cat", exclude_datetime=False + ) == (["date_cat"], []) diff --git a/tests/test_variable_handling/test_remove_variables.py b/tests/test_variable_handling/test_remove_variables.py deleted file mode 100644 index 3984d2c45..000000000 --- a/tests/test_variable_handling/test_remove_variables.py +++ /dev/null @@ -1,28 +0,0 @@ -import pandas as pd -import pytest - -from feature_engine.variable_handling.retain_variables import retain_variables_if_in_df - -test_dict = [ - ( - pd.DataFrame(columns=["A", "B", "C", "D", "E"]), - ["A", "C", "B", "G", "H"], - ["A", "C", "B"], - ["X", "Y"], - ), - (pd.DataFrame(columns=[1, 2, 3, 4, 5]), [1, 2, 4, 6], [1, 2, 4], [6, 7]), - (pd.DataFrame(columns=[1, 2, 3, 4, 5]), 1, [1], 7), - (pd.DataFrame(columns=["A", "B", "C", "D", "E"]), "C", ["C"], "G"), -] - - -@pytest.mark.parametrize("df, variables, overlap, col_not_in_df", test_dict) -def test_retain_variables_if_in_df(df, variables, overlap, col_not_in_df): - - msg = "None of the variables in the list are present in the dataframe." - - assert retain_variables_if_in_df(df, variables) == overlap - - with pytest.raises(ValueError) as record: - retain_variables_if_in_df(df, col_not_in_df) - assert str(record.value) == msg diff --git a/tests/test_variable_handling/test_retain_variables.py b/tests/test_variable_handling/test_retain_variables.py new file mode 100644 index 000000000..3b839d73f --- /dev/null +++ b/tests/test_variable_handling/test_retain_variables.py @@ -0,0 +1,39 @@ +import pandas as pd +import polars as pl +import pytest + +from feature_engine.variable_handling.retain_variables import retain_variables_if_in_df + +test_dict = [ + (["A", "C", "B", "G", "H"], ["A", "C", "B"], ["X", "Y"]), + ("C", ["C"], "G"), +] + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +@pytest.mark.parametrize("variables, overlap, col_not_in_df", test_dict) +def test_retain_variables_if_in_df(make_df, variables, overlap, col_not_in_df): + df = make_df({"A": [1], "B": [1], "C": [1], "D": [1], "E": [1]}) + + msg = "None of the variables in the list are present in the dataframe." + + assert retain_variables_if_in_df(df, variables) == overlap + + with pytest.raises(ValueError, match=msg): + retain_variables_if_in_df(df, col_not_in_df) + + +def test_retain_variables_if_in_df_int_column_names(): + # polars requires string column names. int-named columns are pandas-only + df = pd.DataFrame({1: [1], 2: [1], 3: [1], 4: [1], 5: [1]}) + + msg = "None of the variables in the list are present in the dataframe." + + assert retain_variables_if_in_df(df, [1, 2, 4, 6]) == [1, 2, 4] + assert retain_variables_if_in_df(df, 1) == [1] + + with pytest.raises(ValueError, match=msg): + retain_variables_if_in_df(df, [6, 7]) + + with pytest.raises(ValueError, match=msg): + retain_variables_if_in_df(df, 7) diff --git a/tests/test_variable_handling/test_variable_type_checks.py b/tests/test_variable_handling/test_variable_type_checks.py new file mode 100644 index 000000000..e09da4438 --- /dev/null +++ b/tests/test_variable_handling/test_variable_type_checks.py @@ -0,0 +1,246 @@ +from datetime import date + +import narwhals as nw +import pandas as pd +import polars as pl + +from feature_engine.variable_handling._variable_type_checks import ( + _is_categorical_and_is_datetime, + _is_categorical_and_is_not_datetime, + _is_categories_num, + _is_convertible_to_dt, + _is_convertible_to_num, + _is_date_or_datetime, + _looks_like_date_string, +) + + +def nw_series(values, dtype=None): + s = pl.Series("x", values) + if dtype is not None: + s = s.cast(dtype) + return nw.from_native(s, series_only=True) + + +def nw_pandas_series(values, dtype=None): + s = pd.Series(values, dtype=dtype) + return nw.from_native(s, series_only=True) + + +def test_is_date_or_datetime(): + """A dtype is a date or datetime if it is narwhals' Date or Datetime type.""" + assert _is_date_or_datetime(nw_series([date(2020, 1, 1)]).dtype) is True + assert ( + _is_date_or_datetime(nw_series(["2020-01-01"]).str.to_datetime().dtype) + is True + ) + assert _is_date_or_datetime(nw_series(["a", "b"]).dtype) is False + assert _is_date_or_datetime(nw_series([1, 2, 3]).dtype) is False + + +def test_looks_like_date_string(): + """A string looks like a date if dateutil finds at least 2 date/time fields + in it - this rejects bare numbers that dateutil would otherwise happily + "parse" as a single field (e.g. a day), while still accepting real dates + in non-ISO formats and bare times. + """ + # real dates, including non-ISO formats + assert _looks_like_date_string("2020-01-01") is True + assert _looks_like_date_string("01-Jan-2010") is True + assert _looks_like_date_string("10/11/12") is True + + # bare times + assert _looks_like_date_string("21:45:23") is True + assert _looks_like_date_string("08:00") is True + + # partial dates + assert _looks_like_date_string("Jan 2020") is True + + # bare numbers dateutil could misparse as a single date/time field + assert _looks_like_date_string("20") is False + assert _looks_like_date_string("1999") is False + assert _looks_like_date_string("12") is False + + # non-date garbage + assert _looks_like_date_string("hello") is False + assert _looks_like_date_string("") is False + + # non-string values (e.g. from a mixed-type pandas Object column) must not + # raise, they simply aren't date strings + assert _looks_like_date_string(20) is False + assert _looks_like_date_string(1.5) is False + assert _looks_like_date_string(None) is False + + +def test_is_convertible_to_num(): + """A series is convertible to numeric if every non-null value can be cast + to float. + """ + assert _is_convertible_to_num(nw_series(["20", "21", "19"])) is True + assert _is_convertible_to_num(nw_series(["a", "b"])) is False + assert ( + _is_convertible_to_num(nw_series(["20", "21"], dtype=pl.Categorical)) + is True + ) + + # object dtype columns (pandas-only concept - narwhals classifies a plain + # object dtype column of ints as `nw.Object`, not `nw.String`) + assert _is_convertible_to_num(nw_pandas_series([1, 2], dtype="object")) is True + assert ( + _is_convertible_to_num( + nw_pandas_series([pd.Timestamp("2020-01-01")], dtype="object") + ) + is False + ) + + +def test_is_convertible_to_dt(): + """A series is convertible to datetime if every non-null value is either a + real date/datetime object, or a string that looks like a date. + """ + assert _is_convertible_to_dt(nw_series(["2020-01-01", "2020-01-02"])) is True + assert _is_convertible_to_dt(nw_series(["a", "b"])) is False + assert _is_convertible_to_dt(nw_series(["20", "21"])) is False + + # flexible, dateutil-backed date guessing works for every backend now, not + # just pandas - so non-ISO formats are recognised here too + assert _is_convertible_to_dt(nw_series(["01-Jan-2010"])) is True + assert _is_convertible_to_dt(nw_series(["10/11/12"])) is True + + # an object dtype column holding actual datetime objects (e.g. pandas + # Timestamps) is trivially convertible, without needing to parse anything + assert ( + _is_convertible_to_dt( + nw_pandas_series([pd.Timestamp("2020-01-01")], dtype="object") + ) + is True + ) + + +def test_is_categories_num(): + """A categorical series' categories are numeric if their dtype is numeric - + only possible for pandas, since polars categories are always string-backed. + """ + non_numeric_cat = nw_series(["a", "b", "c"], dtype=pl.Categorical) + assert _is_categories_num(non_numeric_cat) is False + + numeric_cat = nw_pandas_series([20, 21, 19, 18], dtype="category") + assert _is_categories_num(numeric_cat) is True + + +def test_is_categorical_and_is_datetime(): + """A series is categorical-and-datetime if it is a Categorical/String/Object + column whose values are dates, but not an Enum (an explicit category set is + never treated as a datetime) or a numeric-backed categorical. + """ + assert ( + _is_categorical_and_is_datetime( + nw_series(["2020-01-01", "2020-01-02"], dtype=pl.Categorical) + ) + is True + ) + assert ( + _is_categorical_and_is_datetime(nw_series(["a", "b"], dtype=pl.Categorical)) + is False + ) + assert _is_categorical_and_is_datetime(nw_series(["2020-01-01"])) is True + assert _is_categorical_and_is_datetime(nw_series(["20", "21"])) is False + assert _is_categorical_and_is_datetime(nw_series(["a", "b"])) is False + + # an explicit Enum is always treated as categorical, never as datetime + enum_dtype = pl.Enum(["2020-01-01", "2020-01-02"]) + assert ( + _is_categorical_and_is_datetime( + nw_series(["2020-01-01", "2020-01-02"], dtype=enum_dtype) + ) + is False + ) + + # numeric should be False + assert _is_categorical_and_is_datetime(nw_series([1, 2, 3])) is False + + # a numeric-backed categorical (pandas-only - polars categories are always + # string-backed) can never be a datetime, regardless of the categories + numeric_cat = nw_pandas_series([20, 21, 19, 18], dtype="category") + assert _is_categorical_and_is_datetime(numeric_cat) is False + + # a string-dtype pandas column with datetime-like values + assert ( + _is_categorical_and_is_datetime( + nw_pandas_series(["2020-01-01", "2020-01-02"], dtype="string") + ) + is True + ) + + # object dtype column holding actual Timestamp objects + assert ( + _is_categorical_and_is_datetime( + nw_pandas_series([pd.Timestamp("2020-01-01")], dtype="object") + ) + is True + ) + + # object dtype column holding plain ints - not a datetime + assert ( + _is_categorical_and_is_datetime(nw_pandas_series([1, 2], dtype="object")) + is False + ) + + +def test_is_categorical_and_is_not_datetime(): + """A series is categorical-and-not-datetime if it is a Categorical/String/ + Object/Enum column whose values are not dates. + """ + assert ( + _is_categorical_and_is_not_datetime( + nw_series(["2020-01-01", "2020-01-02"], dtype=pl.Categorical) + ) + is False + ) + assert ( + _is_categorical_and_is_not_datetime( + nw_series(["a", "b"], dtype=pl.Categorical) + ) + is True + ) + assert _is_categorical_and_is_not_datetime(nw_series(["2020-01-01"])) is False + assert _is_categorical_and_is_not_datetime(nw_series(["20", "21"])) is True + assert _is_categorical_and_is_not_datetime(nw_series(["a", "b"])) is True + + # an explicit Enum is always treated as categorical + assert ( + _is_categorical_and_is_not_datetime( + nw_series(["a", "b"], dtype=pl.Enum(["a", "b"])) + ) + is True + ) + + # numeric should be False + assert _is_categorical_and_is_not_datetime(nw_series([1, 2, 3])) is False + + # a numeric-backed categorical is categorical-and-not-datetime + numeric_cat = nw_pandas_series([20, 21, 19, 18], dtype="category") + assert _is_categorical_and_is_not_datetime(numeric_cat) is True + + # object dtype column of plain ints + assert ( + _is_categorical_and_is_not_datetime(nw_pandas_series([1, 2], dtype="object")) + is True + ) + + # object dtype column holding actual Timestamp objects - is a datetime, so + # not "categorical and not datetime" + assert ( + _is_categorical_and_is_not_datetime( + nw_pandas_series([pd.Timestamp("2020-01-01")], dtype="object") + ) + is False + ) + + # string-dtype pandas column not convertible to numeric or datetime + assert ( + _is_categorical_and_is_not_datetime( + nw_pandas_series(["a", "b"], dtype="string") + ) + is True + ) diff --git a/tests/test_wrappers/test_check_estimator_wrappers.py b/tests/test_wrappers/test_check_estimator_wrappers.py index f663ad7b5..cee500d43 100644 --- a/tests/test_wrappers/test_check_estimator_wrappers.py +++ b/tests/test_wrappers/test_check_estimator_wrappers.py @@ -1,10 +1,8 @@ import pandas as pd import pytest -import sklearn from sklearn.impute import SimpleImputer from sklearn.preprocessing import OrdinalEncoder, StandardScaler from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.fixes import parse_version from feature_engine.wrappers import SklearnWrapper from tests.estimator_checks.estimator_checks import ( @@ -17,22 +15,14 @@ check_numerical_variables_assignment, ) -sklearn_version = parse_version(parse_version(sklearn.__version__).base_version) -if sklearn_version < parse_version("1.6"): - - def test_sklearn_transformer_wrapper(): - check_estimator(SklearnWrapper(transformer=SimpleImputer())) - -else: - - def test_sklearn_transformer_wrapper(): - check_estimator( - estimator=SklearnWrapper(transformer=SimpleImputer()), - expected_failed_checks=SklearnWrapper( - transformer=SimpleImputer() - )._more_tags()["_xfail_checks"], - ) +def test_sklearn_transformer_wrapper(): + check_estimator( + estimator=SklearnWrapper(transformer=SimpleImputer()), + expected_failed_checks=SklearnWrapper( + transformer=SimpleImputer() + )._more_tags()["_xfail_checks"], + ) @pytest.mark.parametrize( diff --git a/tests/test_wrappers/test_sklearn_wrapper.py b/tests/test_wrappers/test_sklearn_wrapper.py index f15063cd9..76e816c7e 100644 --- a/tests/test_wrappers/test_sklearn_wrapper.py +++ b/tests/test_wrappers/test_sklearn_wrapper.py @@ -60,13 +60,7 @@ def _OneHotEncoder(sparse, drop=None, dtype=np.float64) -> OneHotEncoder: - """OneHotEncoder sparse argument has been renamed as sparse_output - in scikitlearn >=1.2""" - - if skl_version.split(".")[0] == "1" and int(skl_version.split(".")[1]) >= 2: - return OneHotEncoder(sparse_output=sparse, drop=drop, dtype=dtype) - else: - return OneHotEncoder(sparse=sparse, drop=drop, dtype=dtype) + return OneHotEncoder(sparse_output=sparse, drop=drop, dtype=dtype) @pytest.mark.parametrize( diff --git a/tox.ini b/tox.ini index c31b3edd1..4a21bd8ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,5 @@ [tox] envlist = - py39 - py310 - py311-sklearn150 - py311-sklearn160 py311-sklearn170 py312-pandas230 py312-pandas300 @@ -32,14 +28,6 @@ commands = # Python versions # ------------------------- -[testenv:py39] -deps = - .[tests] - -[testenv:py310] -deps = - .[tests] - [testenv:py313] deps = .[tests] @@ -53,16 +41,6 @@ deps = # scikit-learn matrix # ------------------------- -[testenv:py311-sklearn150] -deps = - .[tests] - scikit-learn==1.5.1 - -[testenv:py311-sklearn160] -deps = - .[tests] - scikit-learn==1.6.1 - [testenv:py311-sklearn170] deps = .[tests]