diff --git a/docs/user_guide/datetime/DatetimeFeatures.rst b/docs/user_guide/datetime/DatetimeFeatures.rst
index 54667f8d3..0ff78f2ae 100644
--- a/docs/user_guide/datetime/DatetimeFeatures.rst
+++ b/docs/user_guide/datetime/DatetimeFeatures.rst
@@ -743,6 +743,60 @@ In the following output we see the resulting dataframe:
As you can see, we do not have the constant features in the transformed dataset.
+With polars
+-----------
+
+:class:`DatetimeFeatures()` also works with polars dataframes, and with any other
+dataframe library supported by `narwhals `_.
+
+.. code:: python
+
+ import polars as pl
+ from feature_engine.datetime import DatetimeFeatures
+
+ toy_df = pl.DataFrame({
+ "id": [1, 2, 3, 4],
+ "var_date": ["2012-06-21", "1998-02-10", "2010-08-03", "2020-10-31"],
+ })
+
+ dfts = DatetimeFeatures(
+ features_to_extract=["month", "year", "day_of_week", "days_in_month"],
+ )
+
+ df_transf = dfts.fit_transform(toy_df)
+
+ df_transf
+
+We see the new features in the following output:
+
+.. code:: text
+
+ shape: (4, 5)
+ ┌─────┬────────────────┬───────────────┬──────────────────────┬────────────────────────┐
+ │ id ┆ var_date_month ┆ var_date_year ┆ var_date_day_of_week ┆ var_date_days_in_month │
+ │ --- ┆ --- ┆ --- ┆ --- ┆ --- │
+ │ i64 ┆ i8 ┆ i32 ┆ i8 ┆ i8 │
+ ╞═════╪════════════════╪═══════════════╪══════════════════════╪════════════════════════╡
+ │ 1 ┆ 6 ┆ 2012 ┆ 3 ┆ 30 │
+ │ 2 ┆ 2 ┆ 1998 ┆ 1 ┆ 28 │
+ │ 3 ┆ 8 ┆ 2010 ┆ 1 ┆ 31 │
+ │ 4 ┆ 10 ┆ 2020 ┆ 5 ┆ 31 │
+ └─────┴────────────────┴───────────────┴──────────────────────┴────────────────────────┘
+
+.. note::
+
+ For non-pandas input, string columns are parsed with narwhals'
+ `Series.str.to_datetime() `_,
+ which can only infer unambiguous formats, like ISO-8601. For anything else, e.g. day-first
+ dates such as *21/06/2012*, pass an explicit `format`. The `dayfirst`, `yearfirst` and
+ `utc` parameters are pandas-`to_datetime`-only options and have no effect on non-pandas
+ input.
+
+.. note::
+
+ `variables="index"` is only supported when `X` is a pandas dataframe, since only pandas
+ dataframes have an index.
+
Working with different timezones
--------------------------------
diff --git a/feature_engine/datetime/_datetime_constants.py b/feature_engine/datetime/_datetime_constants.py
index f7ef69a20..ec247e591 100644
--- a/feature_engine/datetime/_datetime_constants.py
+++ b/feature_engine/datetime/_datetime_constants.py
@@ -1,3 +1,4 @@
+import narwhals as nw
import numpy as np
FEATURES_SUPPORTED = [
@@ -78,3 +79,105 @@
"minute": lambda x: x.dt.minute,
"second": lambda x: x.dt.second,
}
+
+
+def _nw_quarter(x: nw.Series) -> nw.Series:
+ return ((x.dt.month() - 1) // 3) + 1
+
+
+def _nw_semester(x: nw.Series) -> nw.Series:
+ return (x.dt.month() > 6).cast(nw.Int64()) + 1
+
+
+def _nw_week(x: nw.Series) -> nw.Series:
+ # narwhals has no isocalendar(); the "%V" strftime code (ISO week) round-trips
+ # correctly on every backend tested (pandas, polars) via to_string().
+ return x.dt.to_string("%V").cast(nw.Int64())
+
+
+def _nw_day_of_week(x: nw.Series) -> nw.Series:
+ # narwhals weekday() is 1=Monday..7=Sunday; pandas dayofweek is 0=Monday..6=Sunday.
+ return x.dt.weekday() - 1
+
+
+def _nw_weekend(x: nw.Series) -> nw.Series:
+ return (_nw_day_of_week(x) >= 5).cast(nw.Int64())
+
+
+def _nw_is_month_start(x: nw.Series) -> nw.Series:
+ return x.dt.day() == 1
+
+
+def _nw_is_month_end(x: nw.Series) -> nw.Series:
+ # no days_in_month()/is_month_end() in narwhals: a day belongs to the last
+ # day of its month iff the next day rolls over into a different month.
+ return x.dt.offset_by("1d").dt.month() != x.dt.month()
+
+
+def _nw_month_start(x: nw.Series) -> nw.Series:
+ return _nw_is_month_start(x).cast(nw.Int64())
+
+
+def _nw_month_end(x: nw.Series) -> nw.Series:
+ return _nw_is_month_end(x).cast(nw.Int64())
+
+
+def _nw_quarter_start(x: nw.Series) -> nw.Series:
+ # quarters start in Jan/Apr/Jul/Oct, the only months where month % 3 == 1.
+ return (_nw_is_month_start(x) & (x.dt.month() % 3 == 1)).cast(nw.Int64())
+
+
+def _nw_quarter_end(x: nw.Series) -> nw.Series:
+ # quarters end in Mar/Jun/Sep/Dec, the only months where month % 3 == 0.
+ return (_nw_is_month_end(x) & (x.dt.month() % 3 == 0)).cast(nw.Int64())
+
+
+def _nw_year_start(x: nw.Series) -> nw.Series:
+ return (_nw_is_month_start(x) & (x.dt.month() == 1)).cast(nw.Int64())
+
+
+def _nw_year_end(x: nw.Series) -> nw.Series:
+ return (_nw_is_month_end(x) & (x.dt.month() == 12)).cast(nw.Int64())
+
+
+def _nw_leap_year(x: nw.Series) -> nw.Series:
+ year = x.dt.year()
+ return (((year % 4 == 0) & (year % 100 != 0)) | (year % 400 == 0)).cast(
+ nw.Int64()
+ )
+
+
+def _nw_days_in_month(x: nw.Series) -> nw.Series:
+ # start of month, plus a month, minus a day = last day of the original month;
+ # its day number is the month's length. Handles leap years automatically.
+ return x.dt.truncate("1mo").dt.offset_by("1mo").dt.offset_by("-1d").dt.day()
+
+
+# Narwhals-native equivalents of FEATURES_FUNCTIONS above, used for dataframe
+# backends other than pandas. Kept separate from FEATURES_FUNCTIONS (rather than
+# merged into one dispatch) because roughly a third of these features (week,
+# month_end, quarter_end, quarter_start, year_start, year_end, leap_year,
+# days_in_month) benchmarked 2x-53x slower than pandas-native when run through
+# narwhals on a pandas backend, so pandas keeps its fast, unchanged native path.
+FEATURES_FUNCTIONS_NARWHALS = {
+ "month": lambda x: x.dt.month(),
+ "quarter": _nw_quarter,
+ "semester": _nw_semester,
+ "year": lambda x: x.dt.year(),
+ "week": _nw_week,
+ "day_of_week": _nw_day_of_week,
+ "day_of_month": lambda x: x.dt.day(),
+ "day_of_year": lambda x: x.dt.ordinal_day(),
+ "weekend": _nw_weekend,
+ "month_start": _nw_month_start,
+ "month_end": _nw_month_end,
+ "quarter_start": _nw_quarter_start,
+ "quarter_end": _nw_quarter_end,
+ "year_start": _nw_year_start,
+ "year_end": _nw_year_end,
+ "leap_year": _nw_leap_year,
+ "days_in_month": _nw_days_in_month,
+ "hour": lambda x: x.dt.hour(),
+ "minute": lambda x: x.dt.minute(),
+ "second": lambda x: x.dt.second(),
+}
diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py
index 106d50277..770ea8117 100644
--- a/feature_engine/datetime/datetime.py
+++ b/feature_engine/datetime/datetime.py
@@ -2,9 +2,9 @@
from typing import List, Optional, Union
-import pandas as pd
-from pandas.api.types import is_datetime64_any_dtype as is_datetime
-from pandas.api.types import is_numeric_dtype as is_numeric
+import narwhals as nw
+import narwhals.dependencies as nwd
+from narwhals.typing import IntoDataFrame, IntoSeries
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils.validation import check_is_fitted
@@ -35,6 +35,7 @@
from feature_engine.datetime._datetime_constants import (
FEATURES_DEFAULT,
FEATURES_FUNCTIONS,
+ FEATURES_FUNCTIONS_NARWHALS,
FEATURES_SUFFIXES,
FEATURES_SUPPORTED,
)
@@ -58,8 +59,11 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
new columns to the dataset. DatetimeFeatures can extract datetime information from
existing datetime or object-like variables or from the dataframe index.
- DatetimeFeatures uses `pandas.to_datetime` to convert object variables to datetime
- and pandas.dt to extract the features from datetime.
+ DatetimeFeatures works with pandas, polars, and any other narwhals-supported
+ dataframe. For pandas input, it uses `pandas.to_datetime` and pandas' `.dt`
+ accessor to parse and extract features. For other backends, it uses narwhals'
+ equivalent operations; `dayfirst`, `yearfirst` and `utc` are pandas-only parsing
+ options and have no effect on non-pandas input, so use `format` there instead.
The transformer supports the extraction of the following features:
@@ -93,7 +97,8 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
If None, the transformer will find and select all datetime variables,
including variables of type object that can be converted to datetime.
If "index", the transformer will extract datetime features from the
- index of the dataframe.
+ index of the dataframe. "index" is only supported when `X` is a pandas
+ dataframe, since only pandas dataframes have an index.
{return_empty}
@@ -119,11 +124,14 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
dayfirst: bool, default="False"
Specify a date parse order if arg is str or is list-like. If True, parses
dates with the day first, e.g. 10/11/12 is parsed as 2012-11-10. Same as in
- `pandas.to_datetime`.
+ `pandas.to_datetime`. Only applied when `X` is a pandas dataframe; ignored
+ for other backends, which have no equivalent parsing option.
yearfirst: bool, default="False"
Specify a date parse order if arg is str or is list-like.
- Same as in `pandas.to_datetime`.
+ Same as in `pandas.to_datetime`. Only applied when `X` is a pandas
+ dataframe; ignored for other backends, which have no equivalent parsing
+ option.
- If True parses dates with the year first, e.g. 10/11/12 is parsed as
2010-11-12.
@@ -131,14 +139,19 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
utc: bool, default=None
Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime
- objects as well). Same as in `pandas.to_datetime`.
+ objects as well). Same as in `pandas.to_datetime`. Only applied when `X` is
+ a pandas dataframe; ignored for other backends, which have no equivalent
+ parsing option.
format: str, default None
The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for
more information on choices. If you have variables with different formats pass
“mixed”, to infer the format for each element individually. This is risky,
and you should probably use it along with dayfirst, according to pandas'
- documentation.
+ documentation. For non-pandas input, `format` is passed to narwhals'
+ `Series.str.to_datetime()`, which does not support "mixed"; unlike pandas,
+ it can only infer a common format across the column when `format=None`, and
+ only for unambiguous formats (e.g. ISO-8601).
Attributes
@@ -168,6 +181,7 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
--------
pandas.to_datetime
pandas.dt
+ narwhals.Series.dt
Examples
--------
@@ -182,6 +196,25 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin)
0 2022 9 18
1 2022 10 27
2 2022 12 24
+
+ With polars:
+
+ >>> import polars as pl
+ >>> from feature_engine.datetime import DatetimeFeatures
+ >>> X = pl.DataFrame(dict(date = ["2022-09-18", "2022-10-27", "2022-12-24"]))
+ >>> dtf = DatetimeFeatures(features_to_extract = ["year", "month", "day_of_month"])
+ >>> dtf.fit(X)
+ >>> dtf.transform(X)
+ shape: (3, 3)
+ ┌───────────┬────────────┬───────────────────┐
+ │ date_year ┆ date_month ┆ date_day_of_month │
+ │ --- ┆ --- ┆ --- │
+ │ i32 ┆ i8 ┆ i8 │
+ ╞═══════════╪════════════╪═══════════════════╡
+ │ 2022 ┆ 9 ┆ 18 │
+ │ 2022 ┆ 10 ┆ 27 │
+ │ 2022 ┆ 12 ┆ 24 │
+ └───────────┴────────────┴───────────────────┘
"""
def __init__(
@@ -240,7 +273,7 @@ def __init__(
self.features_to_extract = features_to_extract
self.format = format
- def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
+ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None):
"""
This transformer does not learn any parameter.
@@ -249,23 +282,36 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
Parameters
----------
- X: pandas dataframe of shape = [n_samples, n_features]
+ X: dataframe of shape = [n_samples, n_features]
The training input samples. Can be the entire dataframe, not just the
variables to transform.
- y: pandas Series, default=None
+ y: Series, default=None
It is not needed in this transformer. You can pass y or None.
"""
# check input dataframe
X = check_X(X)
+ is_pandas = nwd.is_pandas_dataframe(X)
# special case index
if self.variables == "index":
+ # polars and other narwhals backends have no index concept.
+ if is_pandas is False:
+ raise TypeError(
+ "variables='index' requires a pandas dataframe, since only "
+ f"pandas dataframes have an index. Got {type(X)} instead."
+ )
+ pd_ = nw.from_native(X, eager_only=True).__native_namespace__()
+ index_is_dt = pd_.api.types.is_datetime64_any_dtype(X.index)
+ index_is_numeric = pd_.api.types.is_numeric_dtype(X.index)
if not (
- is_datetime(X.index)
+ index_is_dt
or (
- not is_numeric(X.index) and _is_categorical_and_is_datetime(X.index)
+ index_is_numeric is False
+ and _is_categorical_and_is_datetime(
+ nw.from_native(pd_.Series(X.index), series_only=True)
+ )
)
):
raise TypeError("The dataframe index is not datetime.")
@@ -295,25 +341,28 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
self.features_to_extract_ = self.features_to_extract
# save input features
- self.feature_names_in_ = X.columns.tolist()
+ if is_pandas is True:
+ self.feature_names_in_ = list(X.columns)
+ else:
+ self.feature_names_in_ = nw.from_native(X, eager_only=True).columns
# save train set shape
self.n_features_in_ = X.shape[1]
return self
- def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+ def transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""
Extract the date and time features and add them to the dataframe.
Parameters
----------
- X: pandas dataframe of shape = [n_samples, n_features]
+ X: dataframe of shape = [n_samples, n_features]
The data to transform.
Returns
-------
- X_new: pandas dataframe, shape = [n_samples, n_features x n_df_features]
+ X_new: dataframe, shape = [n_samples, n_features x n_df_features]
The dataframe with the original variables plus the new variables.
"""
@@ -326,18 +375,29 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame:
# Check if input data contains same number of columns as dataframe used to fit.
_check_X_matches_training_df(X, self.n_features_in_)
+ is_pandas = nwd.is_pandas_dataframe(X)
+
# reorder variables to match train set
- X = X[self.feature_names_in_]
+ if is_pandas is True:
+ X = X[self.feature_names_in_]
+ else:
+ X = (
+ nw.from_native(X, eager_only=True)
+ .select(self.feature_names_in_)
+ .to_native()
+ )
- # special case index
+ # special case index: only reachable for pandas, fit() already raised
+ # TypeError for any other backend, since only pandas has an index.
if self.variables == "index":
# check if dataset contains na
if self.missing_values == "raise":
self._check_index_contains_na(X.index)
+ pd_ = nw.from_native(X, eager_only=True).__native_namespace__()
# convert index to a datetime series
- idx_datetime = pd.Series(
- pd.to_datetime(
+ idx_datetime = pd_.Series(
+ pd_.to_datetime(
X.index,
dayfirst=self.dayfirst,
yearfirst=self.yearfirst,
@@ -359,32 +419,58 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame:
if len(self.variables_) == 0:
return X
- # convert datetime variables
- datetime_df = pd.concat(
- [
- pd.to_datetime(
- X[variable],
- dayfirst=self.dayfirst,
- yearfirst=self.yearfirst,
- utc=self.utc,
- format=self.format,
- )
- for variable in self.variables_
- ],
- axis=1,
- )
+ if is_pandas is True:
+ pd_ = nw.from_native(X, eager_only=True).__native_namespace__()
+ # convert datetime variables
+ datetime_df = pd_.concat(
+ [
+ pd_.to_datetime(
+ X[variable],
+ dayfirst=self.dayfirst,
+ yearfirst=self.yearfirst,
+ utc=self.utc,
+ format=self.format,
+ )
+ for variable in self.variables_
+ ],
+ axis=1,
+ )
- # create new features
- for var in self.variables_:
- for feat in self.features_to_extract_:
- X[str(var) + FEATURES_SUFFIXES[feat]] = FEATURES_FUNCTIONS[feat](
- datetime_df[var]
- )
- if self.drop_original:
- X.drop(self.variables_, axis=1, inplace=True)
+ # create new features
+ for var in self.variables_:
+ for feat in self.features_to_extract_:
+ X[str(var) + FEATURES_SUFFIXES[feat]] = FEATURES_FUNCTIONS[
+ feat
+ ](datetime_df[var])
+ if self.drop_original:
+ X.drop(self.variables_, axis=1, inplace=True)
+ else:
+ # dayfirst/yearfirst/utc are pandas.to_datetime-only knobs with no
+ # narwhals equivalent, so only `format` is honoured on this branch.
+ nw_X = nw.from_native(X, eager_only=True)
+ new_series = [
+ FEATURES_FUNCTIONS_NARWHALS[feat](
+ self._to_nw_datetime(nw_X.get_column(var))
+ ).alias(str(var) + FEATURES_SUFFIXES[feat])
+ for var in self.variables_
+ for feat in self.features_to_extract_
+ ]
+ nw_X = nw_X.with_columns(*new_series)
+ if self.drop_original:
+ nw_X = nw_X.drop(self.variables_)
+ X = nw_X.to_native()
return X
+ def _to_nw_datetime(self, col: nw.Series) -> nw.Series:
+ """Ensure a narwhals Series has Datetime dtype, parsing strings/categoricals
+ and casting bare Dates (whose `.dt` methods reject hour/minute/second)."""
+ if isinstance(col.dtype, nw.Datetime):
+ return col
+ if isinstance(col.dtype, nw.Date):
+ return col.cast(nw.Datetime())
+ return col.cast(nw.String()).str.to_datetime(format=self.format)
+
def _get_new_features_name(self) -> List:
"""create the names for the datetime features."""
@@ -401,7 +487,7 @@ def _get_new_features_name(self) -> List:
return feature_names
- def _check_index_contains_na(self, index: pd.Index):
+ def _check_index_contains_na(self, index) -> None:
if index.isnull().any():
raise ValueError(
"The dataframe index contains missing data. "
diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py
index acd9d06e8..dbd6285ef 100644
--- a/tests/test_datetime/test_datetime_features.py
+++ b/tests/test_datetime/test_datetime_features.py
@@ -1,5 +1,7 @@
+import narwhals as nw
import numpy as np
import pandas as pd
+import polars as pl
import pytest
from sklearn.exceptions import NotFittedError
from sklearn.pipeline import Pipeline
@@ -7,6 +9,7 @@
from feature_engine.datetime import DatetimeFeatures
from feature_engine.datetime._datetime_constants import (
FEATURES_DEFAULT,
+ FEATURES_FUNCTIONS,
FEATURES_SUFFIXES,
FEATURES_SUPPORTED,
)
@@ -23,6 +26,39 @@
index=pd.date_range("2003-02-27", periods=4, freq="D"),
)
+# ISO-8601 strings parse identically on pandas and polars/narwhals (unlike the
+# dateutil-style formats in df_datetime above, which are pandas-only), so these
+# back the cross-backend tests. Covers a leap day and a year/quarter/month
+# boundary, so "all" features exercise every derived (non-1:1) narwhals feature.
+CROSS_BACKEND_DATES = [
+ "2020-01-01 00:00:00",
+ "2020-02-29 12:30:45",
+ "2020-12-31 23:59:59",
+ "2021-07-15 06:07:08",
+]
+CROSS_BACKEND_DATA = {
+ "Name": ["tom", "nick", "krish", "jack"],
+ "Age": [20, 21, 19, 18],
+ "date": CROSS_BACKEND_DATES,
+}
+feat_names_default_cb = [f"date{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT]
+
+
+def _expected_cross_backend_features(feats):
+ """Reference feature values computed with pandas' native FEATURES_FUNCTIONS,
+ the ground truth both the pandas and the narwhals extraction paths must match."""
+ dt = pd.Series(pd.to_datetime(CROSS_BACKEND_DATES))
+ return {
+ f"date{FEATURES_SUFFIXES[feat]}": list(FEATURES_FUNCTIONS[feat](dt))
+ for feat in feats
+ }
+
+
+def _to_py_values(column):
+ # normalise pandas/numpy and polars scalar containers to plain Python ints
+ # so the two backends' outputs compare equal regardless of dtype width.
+ return [int(v) for v in column]
+
_false_input_params = [
(["not_supported"], 3.519, "wrong_option"),
@@ -173,72 +209,46 @@ def test_raises_non_fitted_error(df_datetime):
DatetimeFeatures().transform(df_datetime)
-def test_extract_datetime_features_with_default_options(
- df_datetime, df_datetime_transformed
-):
- transformer = DatetimeFeatures()
- X = transformer.fit_transform(df_datetime)
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt + [var + feat for var in vars_dt for feat in feat_names_default]
- ],
- check_dtype=False,
- )
-
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_extract_datetime_features_with_default_options(make_df):
+ X = make_df(CROSS_BACKEND_DATA)
+ Xt = DatetimeFeatures().fit_transform(X)
-def test_extract_datetime_features_from_specified_variables(
- df_datetime, df_datetime_transformed
-):
- # single datetime variable
- X = DatetimeFeatures(variables="date_obj1").fit_transform(df_datetime)
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt
- + ["datetime_range", "date_obj2", "time_obj"]
- + ["date_obj1" + feat for feat in feat_names_default]
- ],
- check_dtype=False,
- )
+ result = nw.from_native(Xt, eager_only=True)
+ assert result.columns == vars_non_dt + feat_names_default_cb
+ for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items():
+ assert _to_py_values(result.get_column(col)) == expected
- # multiple datetime variables
- X = DatetimeFeatures(variables=["datetime_range", "date_obj2"]).fit_transform(
- df_datetime
- )
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt
- + ["date_obj1", "time_obj"]
- + [
- var + feat
- for var in ["datetime_range", "date_obj2"]
- for feat in feat_names_default
- ]
- ],
- check_dtype=False,
- )
- # multiple datetime variables in different order than they appear in the df
- X = DatetimeFeatures(variables=["date_obj2", "date_obj1"]).fit_transform(
- df_datetime
- )
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt
- + ["datetime_range", "time_obj"]
- + [
- var + feat
- for var in ["date_obj2", "date_obj1"]
- for feat in feat_names_default
- ]
- ],
- check_dtype=False,
- )
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_extract_datetime_features_from_specified_variables(make_df):
+ data = dict(CROSS_BACKEND_DATA)
+ data["date2"] = CROSS_BACKEND_DATES
+ X = make_df(data)
- # datetime variable is index
+ # single datetime variable
+ Xt = DatetimeFeatures(variables="date").fit_transform(X)
+ result = nw.from_native(Xt, eager_only=True)
+ assert result.columns == vars_non_dt + ["date2"] + feat_names_default_cb
+ for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items():
+ assert _to_py_values(result.get_column(col)) == expected
+
+ # multiple datetime variables, in different order than they appear in X
+ Xt = DatetimeFeatures(variables=["date2", "date"]).fit_transform(X)
+ result = nw.from_native(Xt, eager_only=True)
+ expected_cols = vars_non_dt + [
+ f"date2{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT
+ ] + feat_names_default_cb
+ assert result.columns == expected_cols
+ for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items():
+ assert _to_py_values(result.get_column(col)) == expected
+ assert _to_py_values(result.get_column(col.replace("date", "date2"))) == (
+ expected
+ )
+
+
+def test_extract_datetime_features_from_index():
+ # "index" is pandas-only: polars and other narwhals backends have no index.
X = DatetimeFeatures(
variables="index", features_to_extract=["month", "day_of_month"]
).fit_transform(dates_idx_dt)
@@ -259,38 +269,43 @@ def test_extract_datetime_features_from_specified_variables(
)
-def test_extract_all_datetime_features(df_datetime, df_datetime_transformed):
- X = DatetimeFeatures(features_to_extract="all").fit_transform(df_datetime)
- pd.testing.assert_frame_equal(
- X, df_datetime_transformed.drop(vars_dt, axis=1), check_dtype=False
- )
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_variables_index_raises_on_non_pandas(make_df):
+ X = make_df(CROSS_BACKEND_DATA)
+ transformer = DatetimeFeatures(variables="index")
+ if make_df is pd.DataFrame:
+ with pytest.raises(TypeError, match="The dataframe index is not datetime."):
+ transformer.fit(X)
+ else:
+ with pytest.raises(TypeError, match="variables='index' requires a pandas"):
+ transformer.fit(X)
-def test_extract_specified_datetime_features(df_datetime, df_datetime_transformed):
- X = DatetimeFeatures(features_to_extract=["semester", "week"]).fit_transform(
- df_datetime
- )
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt
- + [var + "_" + feat for var in vars_dt for feat in ["semester", "week"]]
- ],
- check_dtype=False,
- )
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_extract_all_datetime_features(make_df):
+ X = make_df(CROSS_BACKEND_DATA)
+ Xt = DatetimeFeatures(features_to_extract="all").fit_transform(X)
+
+ result = nw.from_native(Xt, eager_only=True)
+ expected = _expected_cross_backend_features(FEATURES_SUPPORTED)
+ assert result.columns == vars_non_dt + list(expected.keys())
+ for col, values in expected.items():
+ assert _to_py_values(result.get_column(col)) == values
- # different order than they appear in the glossary
- X = DatetimeFeatures(features_to_extract=["hour", "day_of_week"]).fit_transform(
- df_datetime
- )
- pd.testing.assert_frame_equal(
- X,
- df_datetime_transformed[
- vars_non_dt
- + [var + "_" + feat for var in vars_dt for feat in ["hour", "day_of_week"]]
- ],
- check_dtype=False,
- )
+
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+@pytest.mark.parametrize(
+ "features", [["semester", "week"], ["hour", "day_of_week"]]
+)
+def test_extract_specified_datetime_features(make_df, features):
+ X = make_df(CROSS_BACKEND_DATA)
+ Xt = DatetimeFeatures(features_to_extract=features).fit_transform(X)
+
+ result = nw.from_native(Xt, eager_only=True)
+ expected = _expected_cross_backend_features(features)
+ assert result.columns == vars_non_dt + list(expected.keys())
+ for col, values in expected.items():
+ assert _to_py_values(result.get_column(col)) == values
def test_extract_features_from_categorical_variable(
@@ -418,43 +433,53 @@ def test_extract_features_from_localized_tz_variables():
pd.testing.assert_frame_equal(X, df_expected, check_dtype=False)
-def test_extract_features_without_dropping_original_variables(
- df_datetime, df_datetime_transformed
-):
- X = DatetimeFeatures(
- variables=["datetime_range", "date_obj2"],
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_extract_features_without_dropping_original_variables(make_df):
+ data = dict(CROSS_BACKEND_DATA)
+ data["date2"] = CROSS_BACKEND_DATES
+ X = make_df(data)
+
+ Xt = DatetimeFeatures(
+ variables=["date", "date2"],
features_to_extract=["week", "quarter"],
drop_original=False,
- ).fit_transform(df_datetime)
-
- pd.testing.assert_frame_equal(
- X,
- pd.concat(
- [df_datetime_transformed[column] for column in vars_non_dt]
- + [df_datetime[var] for var in vars_dt]
- + [
- df_datetime_transformed[feat]
- for feat in [
- var + "_" + feat
- for var in ["datetime_range", "date_obj2"]
- for feat in ["week", "quarter"]
- ]
- ],
- axis=1,
- ),
- check_dtype=False,
+ ).fit_transform(X)
+
+ result = nw.from_native(Xt, eager_only=True)
+ expected_cols = (
+ vars_non_dt
+ + ["date", "date2"]
+ + [
+ f"{var}{FEATURES_SUFFIXES[feat]}"
+ for var in ["date", "date2"]
+ for feat in ["week", "quarter"]
+ ]
)
+ assert result.columns == expected_cols
+ for col, values in _expected_cross_backend_features(["week", "quarter"]).items():
+ assert _to_py_values(result.get_column(col)) == values
+ assert _to_py_values(result.get_column(col.replace("date", "date2"))) == (
+ values
+ )
+
+
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_extract_features_from_variables_containing_nans(make_df):
+ X = make_df({"dates_na": ["2010-02-01", None, "1922-06-01", None]})
+ Xt = DatetimeFeatures(
+ features_to_extract=["year"], missing_values="ignore"
+ ).fit_transform(X)
+ result = nw.from_native(Xt, eager_only=True).get_column("dates_na_year")
+ values = result.to_list()
+ assert values[0] == 2010 or values[0] == 2010.0
+ assert values[1] is None or (isinstance(values[1], float) and np.isnan(values[1]))
+ assert values[2] == 1922 or values[2] == 1922.0
+ assert values[3] is None or (isinstance(values[3], float) and np.isnan(values[3]))
-def test_extract_features_from_variables_containing_nans():
- X = DatetimeFeatures(
- features_to_extract=["year"], missing_values="ignore"
- ).fit_transform(dates_nan)
- pd.testing.assert_frame_equal(
- X,
- pd.DataFrame({"dates_na_year": [2010, np.nan, 1922, np.nan]}),
- )
- # dt variable is index
+
+def test_extract_features_from_index_containing_nans():
+ # "index" is pandas-only: polars and other narwhals backends have no index.
X = DatetimeFeatures(
variables="index", features_to_extract=["month"], missing_values="ignore"
).fit_transform(dates_idx_nan)
@@ -472,6 +497,26 @@ def test_extract_features_from_variables_containing_nans():
)
+def test_polars_string_parsing_needs_explicit_format_for_ambiguous_dates():
+ # dayfirst/yearfirst are pandas.to_datetime-only: narwhals' generic
+ # str.to_datetime() has no day/year-first heuristic, so an ambiguous,
+ # non-ISO format needs an explicit `format` on non-pandas input.
+ X = pl.DataFrame({"date_obj1": ["01-Jan-2010", "24-Feb-1945"]})
+ transformer = DatetimeFeatures(variables="date_obj1", features_to_extract=["year"])
+ transformer.fit(X)
+ with pytest.raises(Exception, match="could not find an appropriate format"):
+ transformer.transform(X)
+
+ transformer = DatetimeFeatures(
+ variables="date_obj1", features_to_extract=["year"], format="%d-%b-%Y"
+ )
+ transformer.fit(X)
+ Xt = transformer.transform(X)
+ assert nw.from_native(Xt, eager_only=True).get_column(
+ "date_obj1_year"
+ ).to_list() == [2010, 1945]
+
+
def test_extract_features_with_different_datetime_parsing_options(df_datetime):
X = DatetimeFeatures(
features_to_extract=["day_of_month"], dayfirst=True
@@ -492,6 +537,20 @@ def test_extract_features_with_different_datetime_parsing_options(df_datetime):
)
+@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame])
+def test_get_feature_names_out_cross_backend(make_df):
+ X = make_df(CROSS_BACKEND_DATA)
+ transformer = DatetimeFeatures()
+ Xt = transformer.fit_transform(X)
+ result = nw.from_native(Xt, eager_only=True)
+ assert result.columns == transformer.get_feature_names_out()
+
+ transformer = DatetimeFeatures(drop_original=False)
+ Xt = transformer.fit_transform(X)
+ result = nw.from_native(Xt, eager_only=True)
+ assert result.columns == transformer.get_feature_names_out()
+
+
def test_get_feature_names_out(df_datetime, df_datetime_transformed):
# default features from all variables
transformer = DatetimeFeatures()