diff --git a/CHANGELOG.md b/CHANGELOG.md index e165a6b..c76caa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,34 @@ ## Unreleased +`ModelSpec` has moved to its own package, +[modelmatrix](https://github.com/intro-stat-learning/modelmatrix), which +ISLP now depends on (`modelmatrix>=0.3`). + +### Breaking changes + +- **`ModelSpec`'s `intercept` defaults to `False`**, as 0.6.0 announced. Pass + `intercept=True` for statsmodels' models (`OLS`, `GLM`, ...) and for + `sklearn_sm` and `sklearn_selected` with them; most scikit-learn + estimators, and glmstar's, fit the intercept themselves. The + `FutureWarning` of 0.6.0 is gone. + +### Changes + +- `from ISLP.models import ModelSpec` and the other tools that build model + matrices are modelmatrix's, as are `ISLP.models.model_spec`, + `ISLP.models.columns` and `ISLP.transforms`, which now are modelmatrix's + modules, so imports from them keep working. `ISLP.models` also exports + modelmatrix's `wildcard` (R's `.`), `Contrast`, `Ordinal` and its + responses `Surv`, `Binomial`, `Multinomial` and `Outcome`. +- ISLP's tests of `ModelSpec` moved to modelmatrix. + +### Bug fixes + +- Stepwise selection (`Stepwise`, `min_max`) counts and finds terms among + `ModelSpec.terms_`, the fitted terms, not `terms`: with a `wildcard`, a + single term standing for many, the terms were misaligned. + ## 0.6.0 (2026-10-26) `ModelSpec` was audited before it moves to its own package in ISLP 0.7. diff --git a/ISLP/models/__init__.py b/ISLP/models/__init__.py index baef65f..a6111c0 100644 --- a/ISLP/models/__init__.py +++ b/ISLP/models/__init__.py @@ -2,13 +2,18 @@ This package contains tools to specify, manipulate and select regression models. +`ModelSpec` and the features of a model are those of the modelmatrix +package, imported here so that `from ISLP.models import ModelSpec` +keeps working. """ import numpy as np, pandas as pd from io import StringIO -from .model_spec import (ModelSpec, +from modelmatrix import (ModelSpec, Column, Feature, + Contrast, + Ordinal, poly, ns, bs, @@ -16,7 +21,14 @@ pca, contrast, ordinal, - build_columns) + build_columns, + wildcard, + Wildcard, + Response, + Surv, + Binomial, + Multinomial, + Outcome) from .strategy import (min_max as min_max_strategy, Stepwise) from .generic_selector import FeatureSelector diff --git a/ISLP/models/columns.py b/ISLP/models/columns.py index 36f9bdc..6e9d210 100644 --- a/ISLP/models/columns.py +++ b/ISLP/models/columns.py @@ -1,324 +1,10 @@ +""" +Moved to the modelmatrix package, as `modelmatrix.columns`: this module +is `modelmatrix.columns` itself, so that imports from it keep working. +""" -from typing import NamedTuple, Any -from copy import copy +import sys -import numpy as np, pandas as pd -from scipy import sparse as sp_sparse -from sklearn.base import clone -from sklearn.preprocessing import (OneHotEncoder, - OrdinalEncoder) -from sklearn.utils.validation import check_is_fitted -from sklearn.exceptions import NotFittedError +from modelmatrix import columns as _module -class Column(NamedTuple): - - """ - A column extractor with a possible - encoder (following `sklearn` fit/transform template). - """ - - idx: Any - name: str - is_categorical: bool = False - is_ordinal: bool = False - columns: tuple = () - encoder: Any = None - - def get_columns(self, X, fit=False, sparse=False): - - """ - Extract associated column from X, - encoding it with `self.encoder` if not None. - - Parameters - ---------- - - X : array-like - X on which model matrix will be evaluated. - If a `pd.DataFrame` or `pd.Series`, variables that are of - categorical dtype will be treated as categorical. - - fit : bool - If True, fit `self.encoder` on corresponding - column (if it is not already fit). The encoder - must be fit to evaluate the columns: a `Column` - used in the terms of a `ModelSpec` is not itself fit - when the `ModelSpec` is, as a copy of it is. - - sparse : bool - If True, the columns may be returned as a - `scipy.sparse` matrix, e.g. for a categorical encoding. - - Returns - ------- - cols : `np.ndarray` - Evaluated columns -- if an encoder is used, - several columns may be produced. - - names : (str,) - Column names - """ - - cols = _get_column(self.idx, X) - if fit: - self.fit_encoder(X) - - if self.encoder is not None: - transform = self.encoder.transform - if sparse and hasattr(self.encoder, '_islp_sparse_transform'): - transform = self.encoder._islp_sparse_transform - try: - cols = transform(cols) - except NotFittedError as e: - raise NotFittedError( - f"The encoder of Column '{self.name}' is not fit: " - "use get_columns(X, fit=True) to fit it on X.") from e - except ValueError: - # commonly raised for transformers given pd.Seriies - # try casting to a 2-d ndarray - cols = transform(np.asarray(cols).reshape((-1,1))) - else: - cols = _check_numeric(cols, self.name) - if sp_sparse.issparse(cols): - if not sparse: - cols = cols.toarray() - else: - cols = np.asarray(cols) - - names = self.columns - if hasattr(self.encoder, 'columns_'): - names = ['{0}[{1}]'.format(self.name, c) - for c in self.encoder.columns_] - if not names: - names = ['{0}[{1}]'.format(self.name, i) - for i in range(cols.shape[1])] - return cols, names - - def fit_encoder(self, X): - - """ - Fit `self.encoder`. - - Parameters - ---------- - X : array-like - X on which encoder will be fit. - - Returns - ------- - None - """ - cols = _get_column(self.idx, X) - if self.encoder is not None: - try: - check_is_fitted(self.encoder) - except NotFittedError: - self.encoder.fit(cols) - return np.asarray(cols) - - -# private functions - -_NUMERIC_INFERRED = ['integer', 'floating', 'mixed-integer-float', - 'decimal', 'boolean', 'empty'] - -def _check_numeric(col, name): - """ - Raise a `ValueError` if `col` (a column with no encoder) - is not numeric: a string column must be made categorical - explicitly. Returns `col`, as floats if it was of object dtype. - """ - values = np.asarray(col) - if values.dtype.kind in 'biuf': - return col - if values.dtype.kind == 'O': - if pd.api.types.infer_dtype(values.reshape(-1), skipna=True) in _NUMERIC_INFERRED: - return values.astype(float) - raise ValueError( - f"column '{name}' is not numeric; to treat it as categorical, " - f"convert it with `X['{name}'] = X['{name}'].astype('category')` " - f"or use `contrast('{name}')` in the terms") - - -def _get_column(idx, - X, - loc=True): - """ - Extract column `idx` from `X` - as a two-dimensional ndarray or a pd.DataFrame - """ - if isinstance(X, np.ndarray): - col = X[:, [idx]] - elif hasattr(X, 'loc'): - if loc: - col = X.loc[:, [idx]] - else: # use iloc instead - col = X.iloc[:, [idx]] - else: - raise ValueError('expecting an ndarray or a ' + - '"loc/iloc" methods, got %s' % str(X)) - - return col - - -def _get_column_info(X, - columns, - is_categorical, - is_ordinal, - categorical_encoders={} - ): - - - """ - Compute a dictionary - of `Column` instances for each column - of `X`. Keys are `columns`. - - Categorical columns use the encoder given for that - column in `categorical_encoders`, else the one under - 'ordinal' for an ordered column if that key is present, - else the one under 'categorical'. - - """ - - column_info = {} - for i, col in enumerate(columns): - if type(col) == int: - name = f'X{col}' - else: - name = str(col) - if is_categorical[i]: - if col in categorical_encoders: - key = col - elif is_ordinal[i] and 'ordinal' in categorical_encoders: - key = 'ordinal' - else: - key = 'categorical' - encoder = clone(categorical_encoders[key]) - Xcol = _get_column(col, X) - encoder.fit(Xcol) - if hasattr(encoder, 'columns_'): - columns = ['{0}[{1}]'.format(col, c) - for c in encoder.columns_] - else: # only transform to count the columns - cols = encoder.transform(Xcol) - if not sp_sparse.issparse(cols): - cols = np.asarray(cols) - if cols.ndim == 1 or cols.shape[1] == 1: - columns = ['{0}'.format(col)] - else: - columns = ['{0}[{1}]'.format(col, c) - for c in range(cols.shape[1])] - column_info[col] = Column(col, - name, - bool(is_categorical[i]), - bool(is_ordinal[i]), - tuple(columns), - encoder) - else: - Xcol = _get_column(col, X) - column_info[col] = Column(col, - name, - columns=(name,)) - return column_info - -# extracted from method of BaseHistGradientBoosting from -# https://github.com/scikit-learn/scikit-learn/blob/2beed55847ee70d363bdbfe14ee4401438fba057/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py -# max_bins is ignored - -def _check_categories(categorical_features, X): - """Check and validate categorical features in X - - Return - ------ - is_categorical : ndarray of shape (n_features,) or None, dtype=bool - Indicates whether a feature is categorical. If no feature is - categorical, this is None. - known_categories : list of size n_features or None - The list contains, for each feature: - - an array of shape (n_categories,) with the unique cat values - - None if the feature is not categorical - None if no feature is categorical. - """ - if categorical_features is None: - return None, None - - categorical_features = np.asarray(categorical_features) - - if categorical_features.size == 0: - return None, None - - if categorical_features.dtype.kind not in ('i', 'b'): - raise ValueError("categorical_features must be an array-like of " - "bools or array-like of ints.") - - n_features = X.shape[1] - - # check for categorical features as indices - if categorical_features.dtype.kind == 'i': - if (np.max(categorical_features) >= n_features - or np.min(categorical_features) < 0): - raise ValueError("categorical_features set as integer " - "indices must be in [0, n_features - 1]") - is_categorical = np.zeros(n_features, dtype=bool) - is_categorical[categorical_features] = True - else: - if categorical_features.shape[0] != n_features: - raise ValueError("categorical_features set as a boolean mask " - "must have shape (n_features,), got: " - f"{categorical_features.shape}") - is_categorical = categorical_features - - if not np.any(is_categorical): - return None, None - - # compute the known categories in the training data. We need to do - # that here instead of in the BinMapper because in case of early - # stopping, the mapper only gets a fraction of the training data. - known_categories = [] - - for f_idx in range(n_features): - if is_categorical[f_idx]: - # _get_column returns a 2-d column: flatten it to its values - categories = np.array([v for v in - set(np.asarray(_get_column(f_idx, - X, - loc=False)).ravel())]) - missing = [] - for c in categories: - try: - missing.append(np.isnan(c)) - except TypeError: # not a float - missing.append(False) - missing = np.array(missing) - if missing.any(): - categories = sorted(categories[~missing]) - - else: - categories = None - known_categories.append(categories) - - return is_categorical, known_categories - - -def _categorical_from_df(df): - """ - Find - """ - is_categorical = [] - is_ordinal = [] - for c in df.columns: - try: - if df[c].dtype == 'category': - is_categorical.append(True) - is_ordinal.append(df[c].cat.ordered) - else: - is_categorical.append(False) - is_ordinal.append(False) - except (TypeError, AttributeError): - is_categorical.append(False) - is_ordinal.append(False) - is_categorical = np.asarray(is_categorical) - is_ordinal = np.asarray(is_ordinal) - - return is_categorical, is_ordinal +sys.modules[__name__] = _module diff --git a/ISLP/models/model_spec.py b/ISLP/models/model_spec.py index 10ea981..6b1ab71 100644 --- a/ISLP/models/model_spec.py +++ b/ISLP/models/model_spec.py @@ -1,1739 +1,10 @@ """ -Model specification -=================== - -This module defines the basic object to represent regression -formula: *ModelSpec*. +Moved to the modelmatrix package, as `modelmatrix.model_spec`: this module +is `modelmatrix.model_spec` itself, so that imports from it keep working. """ -from collections import namedtuple -from itertools import product -from typing import NamedTuple, Any -from copy import copy -import warnings - -import numpy as np, pandas as pd -import narwhals as nw -from scipy import sparse as sp_sparse - -from sklearn.base import TransformerMixin, BaseEstimator, clone -from sklearn.utils.validation import check_is_fitted -from sklearn.preprocessing import (OneHotEncoder, - OrdinalEncoder, - StandardScaler) -from sklearn.decomposition import PCA -from sklearn.pipeline import make_pipeline, Pipeline -from sklearn.exceptions import NotFittedError -from joblib import hash as joblib_hash - -from .columns import (_get_column_info, - Column, - _categorical_from_df, - _check_categories) - -from ..transforms import (Poly, - BSpline, - NaturalSpline, - Interaction) - -DOCACHE = False - -# ModelSpec will move to its own package: a FutureWarning at fit, with -# advice on `intercept` if the spec relies on its default -_MOVE_WARNING = ( - "ModelSpec will move to its own package in ISLP 0.7. " - "`from ISLP.models import ModelSpec` (and the other tools of ISLP.models " - "that build model matrices) will keep working, but other import paths, " - "such as ISLP.models.model_spec, may not, and `intercept` will then " - "default to False.") -_INTERCEPT_WARNING = ( - " This ModelSpec relies on the default of `intercept`, now True, which adds " - "a column of ones named 'intercept' to the design: only statsmodels' models " - "(OLS, GLM, ...) need it, as glmstar's estimators and most scikit-learn " - "estimators fit the intercept themselves. Pass intercept=True to keep it " - "(e.g. for statsmodels), or intercept=False to drop it.") -_SILENCE_MOVE_WARNING = ( - " To silence this warning: " - "warnings.filterwarnings('ignore', message='ModelSpec will move').") - -class Feature(NamedTuple): - - """ - An element in a model matrix that will build - columns from an array-like X. - """ - - variables: tuple - name: str - encoder: Any - use_transform: bool=True # if False use the predict method - pure_columns: bool=False - override_encoder_colnames: bool=False - # how a helper such as ns() made this Feature: (helper, args, kwargs), - # so that it can be rebuilt with other parameters (ModelSpec.set_params) - spec: Any=None - - -#### contrast specific code - -def _as_series(X): - """ - A single categorical feature as a `pd.Series`. - """ - if isinstance(X, pd.DataFrame): - if X.shape[1] != 1: - raise ValueError(f'expecting a single column, got {X.shape[1]}') - X = X.iloc[:,0] - if not isinstance(X, pd.Series): - X = np.asarray(X) - if X.ndim > 2 or (X.ndim == 2 and X.shape[1] != 1): - raise ValueError(f'expecting a single column, got array of shape {X.shape}') - X = pd.Series(X.reshape(-1)) - return X - -def _observed_levels(X): - """ - Levels of `X` that are observed (missing values excluded), - in the order of `X.dtype.categories` for a categorical, - sorted otherwise. - """ - observed = X.dropna().unique() - if isinstance(X.dtype, pd.CategoricalDtype): - observed = set(observed) - return [c for c in X.dtype.categories if c in observed] - return sorted(observed) - -def _level_codes(X, levels, during='transform'): - """ - Position of each entry of `X` in `levels`, and a mask - of missing entries (whose codes are -1). Raises a `ValueError` - for entries that are neither missing nor in `levels`. - """ - X = _as_series(X) - missing = np.asarray(X.isna()) - codes = pd.Index(levels).get_indexer(np.asarray(X, dtype=object)) - codes[missing] = -1 - unknown = (codes < 0) & ~missing - if unknown.any(): - raise ValueError('Found unknown categories {0} during {1}'.format( - sorted(set(np.asarray(X)[unknown]), key=str), during)) - return codes, missing - -class Contrast(TransformerMixin, BaseEstimator): - - def __init__(self, - method='drop', - drop_level=None): - """ - Contrast encoding for categorical variables. - - Parameters - ---------- - method : ['drop', 'sum', None, callable] - If 'drop', then a column of the one-hot - encoding will be dropped. If 'sum', then the sum of - coefficients is constrained to sum to 1. - If `None`, the full one-hot encoding is returned. - Finally, if callable, then it should take the number of - levels of the category as a single argument and return - an appropriate contrast of the full one-hot encoding. - - drop_level : level identifier (optional) - If not None, this level of the category - will be dropped if `method` is 'drop' (or given - coefficient minus the sum of the others if 'sum'). - Matched against the levels by value, then by `str`. - - Levels are those observed when fit, in the order of - the categories for a categorical dtype and sorted - otherwise; unused categories are dropped. Missing - values give a row of `np.nan`, and a level not seen - when fit raises a `ValueError` on transform. - - """ - - self.method = method - self.drop_level = drop_level - - def fit(self, X, y=None): - - """ - Construct contrast of categorical variable - for use in building a design matrix. - - Parameters - ---------- - X : array-like - X on which model matrix will be evaluated. - If a :py:class:`pd.DataFrame` or :py:class:`pd.Series`, variables that are of - categorical dtype will be treated as categorical. - - Returns - ------- - F : array-like - Columns of design matrix implied by the - categorical variable. - - """ - - # validate before setting any fitted attribute, so a bad method - # does not leave a partially fitted encoder - if not (self.method in ['drop', 'sum', None] or callable(self.method)): - raise ValueError('method must be one of ["drop", "sum", None] or a callable ' + - 'that returns a contrast matrix given the number' + - ' of levels') - - levels = _observed_levels(_as_series(X)) - column_names = [str(l) for l in levels] - nlevel = len(levels) - - if self.method in ['drop', 'sum']: - if self.drop_level is None: # these defaults consistent with R - drop_idx = 0 if self.method == 'drop' else nlevel - 1 - else: - drop_idx = _level_index(levels, self.drop_level) - keep = np.ones(nlevel, bool) - keep[drop_idx] = 0 - columns = [n for n, k in zip(column_names, keep) if k] - contrast = sp_sparse.identity(nlevel, format='csc')[:,keep] - if self.method == 'sum': - contrast = contrast.tolil() - contrast[drop_idx] = -1 - elif callable(self.method): - contrast = sp_sparse.csr_matrix(np.asarray(self.method(nlevel), float)) - columns = ['C({})'.format(i) for i in range(contrast.shape[1])] - elif self.method is None: - contrast = sp_sparse.identity(nlevel, format='csr') - columns = column_names - else: - raise ValueError('method must be one of ["drop", "sum", None] or a callable' + - 'that returns a contrast matrix and column names given the number' + - ' of levels') - - self.levels_ = levels - self.n_features_in_ = 1 - self.columns_ = columns - # stored sparse: a dense (levels x levels) matrix is large - # for a categorical with many levels - self._contrast = sp_sparse.csr_matrix(contrast) - return self - - @property - def contrast_matrix_(self): - # rows are levels, columns are those of the encoding - return self._contrast.toarray() - - def _islp_sparse_transform(self, X): - # the encoding as a sparse matrix, never forming the dense - # (n x levels) indicator matrix - check_is_fitted(self) - codes, missing = _level_codes(X, self.levels_) - rows = np.flatnonzero(~missing) - indicator = sp_sparse.csr_matrix((np.ones(rows.shape[0]), (rows, codes[rows])), - shape=(codes.shape[0], len(self.levels_))) - value = sp_sparse.csr_matrix(indicator @ self._contrast) - if missing.any(): - value = value.tolil() - value[np.flatnonzero(missing)] = np.nan - value = value.tocsr() - return value - - def transform(self, X): - value = self._islp_sparse_transform(X).toarray() - - if isinstance(X, (pd.DataFrame, pd.Series)): - df = pd.DataFrame(value, columns=self.columns_, copy=False) - df.index = X.index - return df - return value - -def _level_index(levels, level): - """ - Index of `level` in `levels`, matched by value then by `str`. - """ - for match in [lambda l: l == level, - lambda l: str(l) == str(level)]: - idx = [i for i, l in enumerate(levels) if match(l)] - if idx: - return idx[0] - raise ValueError('level {0!r} is not among the observed levels {1}'.format( - level, [str(l) for l in levels])) - -class Ordinal(TransformerMixin, BaseEstimator): - - def __init__(self, levels=None): - """ - Ordinal encoding for categorical variables: each entry - is replaced by the position of its level. - - Parameters - ---------- - levels : sequence (optional) - The levels in order. If None, the order must be - unambiguous: the categories of an ordered categorical - dtype (unused categories keep their positions), or - the sorted values of a numeric or boolean column. - Other columns, e.g. strings or an unordered categorical, - raise a `ValueError` since their sorted (alphabetical) - order is rarely the intended one. - - Missing values are encoded as `np.nan`, and a level not - among the levels raises a `ValueError`. - """ - self.levels = levels - - def fit(self, X, y=None): - - """ - Find the levels of the categorical variable. - - Parameters - ---------- - X : array-like - Single column on which the encoding will be evaluated. - - """ - - X = _as_series(X) - if self.levels is not None: - levels = list(self.levels) - if len(set(levels)) != len(levels): - raise ValueError('levels should not contain duplicates') - _level_codes(X, levels, during='fit') # raises if X has other levels - elif isinstance(X.dtype, pd.CategoricalDtype) and X.dtype.ordered: - levels = list(X.dtype.categories) - elif (pd.api.types.is_numeric_dtype(X.dtype) or - pd.api.types.is_bool_dtype(X.dtype)): - levels = _observed_levels(X) - else: - raise ValueError( - 'the order of the levels {0} is ambiguous: give it ' - 'with `levels=[...]`, or use an ordered pd.Categorical, ' - 'e.g. `X[col].cat.as_ordered()` or ' - '`pd.Categorical(..., categories=[...], ordered=True)`'.format( - [str(l) for l in _observed_levels(X)])) - self.levels_ = levels - self.n_features_in_ = 1 - return self - - def transform(self, X): - check_is_fitted(self) - codes, missing = _level_codes(X, self.levels_) - value = codes.astype(float).reshape((-1,1)) - value[missing] = np.nan - return value - -def _term_columns(terms): - """ - The columns of X that `terms` (column identifiers, Columns, - Features and tuples of these) are built from. - """ - found = set() - def visit(t): - if isinstance(t, Feature): - for v in t.variables: - visit(v) - elif isinstance(t, Column): - found.add(t.idx) - elif isinstance(t, tuple): - for v in t: - visit(v) - else: - try: - found.add(t) - except TypeError: # not a column identifier: an error later - pass - for t in terms: - visit(t) - return found - -def _native_columns(X): - # the column names of a non-pandas dataframe, else None - if (isinstance(X, (pd.DataFrame, pd.Series, np.ndarray)) or - sp_sparse.issparse(X)): - return None - try: - X_nw = nw.from_native(X, eager_only=True) - except TypeError: - return None - return list(X_nw.columns) - -def _from_native(X, columns=None): - """ - Convert a dataframe (or series) of any eager backend supported by - narwhals, e.g. polars or pyarrow, to pandas. Returns the converted - `X` and a function converting a `pd.DataFrame` model matrix back to - that backend, or None if `X` was not converted. - - If `columns` is given, only those columns of a dataframe are - converted (the copy is of the columns a model matrix uses). - - Conversion through pandas keeps categoricals, including the order - of a polars Enum or an ordered pyarrow dictionary (as an ordered - `pd.Categorical`). The levels of an unordered categorical, e.g. a - polars Categorical, are stored in order of appearance; they are - sorted, as pandas does when it infers categories, so that the - model matrix does not depend on the order of the rows. - """ - if (isinstance(X, (pd.DataFrame, pd.Series, np.ndarray)) or - sp_sparse.issparse(X)): - return X, None - try: - X_nw = nw.from_native(X, allow_series=True) - except TypeError: # not a dataframe, e.g. a list: left to _check_X - return X, None - if isinstance(X_nw, nw.LazyFrame): - raise TypeError('Lazy frames are not supported by ModelSpec; ' - 'collect them first, e.g. X.collect()') - backend = nw.get_native_namespace(X_nw) - if columns is not None and isinstance(X_nw, nw.DataFrame): - keep = [c for c in X_nw.columns if c in columns] - X_nw = X_nw.select(keep or X_nw.columns[:1]) - - def to_native(D): - duplicated = list(D.columns[D.columns.duplicated()]) - if duplicated: - raise ValueError(f'the model matrix has duplicated column names {duplicated}, ' - 'which this dataframe backend cannot hold') - return nw.from_dict({str(c): D[c].to_numpy() for c in D.columns}, - backend=backend).to_native() - - X = X_nw.to_pandas() - if isinstance(X_nw, nw.Series): - dtypes = {X.name: X_nw.dtype} - else: - dtypes = dict(X_nw.schema) - for name, dtype in dtypes.items(): - col = X if isinstance(X, pd.Series) else X[name] - if not isinstance(col.dtype, pd.CategoricalDtype): - continue - # some versions of polars convert an Enum to an unordered - # pd.Categorical: its categories are in the declared order - if dtype == nw.Enum: - col = col.cat.as_ordered() - elif not col.dtype.ordered: - col = col.cat.reorder_categories(sorted(col.dtype.categories)) - else: - continue - if isinstance(X, pd.Series): - X = col - else: - X[name] = col - return X, to_native - -class ModelSpec(TransformerMixin, BaseEstimator): - - '''Parameters - ---------- - - terms : sequence (optional) - - Sequence of sets whose - elements are columns of *X* when fit. - For :py:class:`pd.DataFrame` these can be column - names. - - intercept : bool or None (optional) - - Include a column of ones named `'intercept'` as the first - column of the design? Pass `True` for statsmodels' models - (`OLS`, `GLM`, ...), which need the column, and `False` for - estimators that fit the intercept themselves (glmstar's - estimators and most scikit-learn estimators). The default, - `None`, behaves as `True` but issues a `FutureWarning` at - *fit*: the default will change to `False` in ISLP 0.7, when - `ModelSpec` moves to its own package. - - categorical_features : array-like of {bool, int} of shape (n_features) - or shape (n_categorical_features,), default=None. - - Indicates the categorical features. Will be ignored if *X* is a :py:class:`pd.DataFrame` - or :py:class:`pd.Series`. - - - None : no feature will be considered categorical for :py:class:`np.ndarray`. - - boolean array-like : boolean mask indicating categorical features. - - integer array-like : integer indices indicating categorical - features. - - categorical_encoders : dict (optional) - - Encoders for the categorical columns of *X*, i.e. the columns of - categorical dtype of a :py:class:`pd.DataFrame` or those given by - *categorical_features* for an :py:class:`np.ndarray`. Its keys are - either: - - - `'categorical'` : the encoder of every categorical column, ordered - or not, without its own entry, cloned for each column. Defaults - to `Contrast(method='drop')`, which drops the first level. - - `'ordinal'` : if present, used instead for the ordered - `pd.Categorical` columns, e.g. `{'ordinal': Ordinal()}` to - score each by the position of its level. No default. - - a column of *X* (a name for a :py:class:`pd.DataFrame`, an integer - for an :py:class:`np.ndarray`) : the encoder of that column. - Like the others it is cloned, and the clone is fit, so the - dictionary's encoders are not modified. - - For an encoder of a single term, see :py:func:`contrast` - and :py:func:`ordinal`. - - sparse_output : bool (optional) - - If True, the model matrix is a `scipy.sparse.csr_matrix`. - Categorical encodings and their interactions are then built - sparse throughout, so a design with categoricals of many levels - never forms its dense columns; other columns (numeric, splines, - etc.) are dense blocks stored sparsely. Not compatible with - `set_output`. - - ''' - - def __init__(self, - terms=(), - intercept=None, - categorical_features=None, - categorical_encoders=None, - sparse_output=False - ): - - self.intercept = intercept - self.terms = terms - self.categorical_features = categorical_features - self.categorical_encoders = categorical_encoders - self.sparse_output = sparse_output - - def _check_X(self, X, reset): - """ - Validate `X`, returning a `pd.DataFrame` or 2-d `np.ndarray`. - - If `reset`, record `n_features_in_` (and `feature_names_in_` - for a `pd.DataFrame`), else check an `np.ndarray` has - `n_features_in_` columns. Columns of a `pd.DataFrame` are - matched by name, so it may have other columns, in any order. - """ - if isinstance(X, pd.Series): - X = X.to_frame() - if sp_sparse.issparse(X): - raise TypeError('Sparse data is not supported by ModelSpec; ' - 'convert it with X.toarray()') - if not isinstance(X, pd.DataFrame): - X = np.asarray(X) - if X.ndim != 2: - raise ValueError('Expected a 2-dimensional array or a pd.DataFrame, ' - f'got an array with {X.ndim} dimension(s). ' - 'Reshape your data either using array.reshape(-1, 1) ' - 'if your data has a single feature or array.reshape(1, -1) ' - 'if it contains a single sample.') - if np.iscomplexobj(X): - raise ValueError('Complex data not supported') - if X.shape[0] == 0: - raise ValueError(f'Found array with 0 sample(s) (shape={X.shape}) ' - 'while a minimum of 1 is required.') - if X.shape[1] == 0: - raise ValueError(f'Found array with 0 feature(s) (shape={X.shape}) ' - 'while a minimum of 1 is required.') - if reset: - self.n_features_in_ = X.shape[1] - if (isinstance(X, pd.DataFrame) and - all(isinstance(c, str) for c in X.columns)): - self.feature_names_in_ = np.asarray(X.columns, dtype=object) - elif hasattr(self, 'feature_names_in_'): - del self.feature_names_in_ - elif (not isinstance(X, pd.DataFrame) and - X.shape[1] != self.n_features_in_): - raise ValueError(f'X has {X.shape[1]} features, but ModelSpec ' - f'is expecting {self.n_features_in_} features as input.') - return X - - # the parameters of the terms, e.g. for GridSearchCV - - def get_params(self, deep=True): - """ - Get parameters of this ModelSpec. - - With `deep`, these include the parameters of each term, as - `'terms____'` for the term at position `i` of - `terms` (`'terms______'` for part `j` of an - interaction), e.g. `'terms__0__df'` for `ns('x', df=4)`. - """ - params = super().get_params(deep=deep) - # parameters are only validated at fit, so terms may be anything here - if deep and isinstance(self.terms, (list, tuple)): - for i, term in enumerate(self.terms): - for key, value in _term_params(term).items(): - params[f'terms__{i}__{key}'] = value - return params - - def set_params(self, **params): - """ - Set parameters of this ModelSpec. - - A parameter of a term is set with `'terms____'`, - where `i` is the position of the term in `terms` or the name - given to it, e.g. `'terms__price__df'` for - `ns('Price', df=4, name='price')`. A term made by `poly()`, - `ns()`, `bs()` or `pca()` is made again with the new parameter, - so its name (if not given) and its columns' names follow it. - `terms` itself is not modified: it is replaced by a copy, in - which every occurrence of the changed term (e.g. also in an - interaction) is changed. - """ - term_params = {k: v for k, v in params.items() if k.startswith('terms__')} - rest = {k: v for k, v in params.items() if not k.startswith('terms__')} - if 'terms' in rest: # a new terms first, then its parameters - super().set_params(terms=rest.pop('terms')) - if term_params: - terms = list(self.terms) - for key, value in term_params.items(): - path = key.split('__')[1:] - i = _term_position(terms, path[0]) - new_term, (old, new) = _set_term_param(terms[i], path[1:], value) - terms[i] = new_term - terms = [_substitute(t, old, new) for t in terms] - container = list if isinstance(self.terms, list) else tuple - super().set_params(terms=container(terms)) - if rest: - super().set_params(**rest) - return self - - # missing values in X propagate to the model matrix - - def __sklearn_tags__(self): - tags = super().__sklearn_tags__() - tags.input_tags.allow_nan = True - return tags - - def _more_tags(self): # scikit-learn < 1.6 - return {'allow_nan': True} - - def _include_intercept(self): - # `intercept=None`, the deprecated default, behaves as True - return self.intercept is None or bool(self.intercept) - - def fit(self, X, y=None): - - """ - Fit the encoders of each term, determining - the columns of the model matrix. - - Parameters - ---------- - X : array-like - X on which model matrix will be evaluated. - If a :py:class:`pd.DataFrame` or :py:class:`pd.Series`, variables that are of - categorical dtype will be treated as categorical. Dataframes of other - backends supported by `narwhals`, such as `polars` or `pyarrow`, are - converted to `pandas`; `transform` then returns the model matrix - in the backend of its input. - - y : None - Ignored. This parameter exists only for compatibility with - :py:class:`sklearn.pipeline.Pipeline`. - - """ - - # warn at fit, not in __init__, as sklearn's clone re-runs __init__ - warnings.warn(_MOVE_WARNING + (_INTERCEPT_WARNING if self.intercept is None - else _SILENCE_MOVE_WARNING), - FutureWarning, stacklevel=2) - - if not isinstance(self.sparse_output, (bool, np.bool_)): - raise ValueError(f'sparse_output should be True or False, got {self.sparse_output!r}') - native_columns = _native_columns(X) - X, _ = _from_native(X, _term_columns(self.terms)) - X = self._check_X(X, reset=True) - if native_columns is not None: # describe all of X, not only the columns used - self.n_features_in_ = len(native_columns) - self.feature_names_in_ = np.asarray(native_columns, dtype=object) - - # built at fit, not in __init__, so set_params(categorical_encoders=...) - # takes effect; update(dict) as the keys may be integers - self.categorical_encoders_ = {'categorical': Contrast(method='drop')} - if self.categorical_encoders is not None: - self.categorical_encoders_.update(self.categorical_encoders) - - if isinstance(X, (pd.DataFrame, pd.Series)): - (categorical_features, - self.is_ordinal_) = _categorical_from_df(X) - (self.is_categorical_, - self.known_categories_) = _check_categories(categorical_features, - X) - self.columns_ = X.columns - if self.is_categorical_ is None: - self.is_categorical_ = np.zeros(X.shape[1], bool) - self.is_ordinal_ = pd.Series(self.is_ordinal_, - index=self.columns_) - self.is_categorical_ = pd.Series(self.is_categorical_, - index=self.columns_) - else: - categorical_features = self.categorical_features - (self.is_categorical_, - self.known_categories_) = _check_categories(categorical_features, - X) - if self.is_categorical_ is None: - self.is_categorical_ = np.zeros(X.shape[1], bool) - self.is_ordinal_ = np.zeros(self.is_categorical_.shape, - bool) - self.columns_ = np.arange(X.shape[1]) - - self.features_ = {} - self.encoders_ = {} - - # encoders are fit on clones so that `self.terms` is never - # mutated and the spec can be refit (or cloned and refit) - clone_memo = {} - - self.column_info_ = _get_column_info(X, - self.columns_, - np.asarray(self.is_categorical_), - np.asarray(self.is_ordinal_), - categorical_encoders=self.categorical_encoders_) - - # include each column as a Feature - # so that their columns are built if needed - - for col_ in self.columns_: - self.features_[col_] = Feature((col_,), str(col_), None, pure_columns=True) - - # find possible interactions and other features - - tmp_cache = {} - - # the encoders are fit on all of X, but the names and number of - # the columns of each term only need its fitted encoders: they are - # found by evaluating the terms on the first rows of X, so that - # fit does not build the model matrix - X_head = X.iloc[:2] if isinstance(X, pd.DataFrame) else X[:2] - - for term in self.terms: - if isinstance(term, Feature): - term_ = _clone_term(term, clone_memo) - self.features_[term] = term_ - build_columns(self.column_info_, - X, - term_, - encoders=self.encoders_, - col_cache=tmp_cache, - fit=True, sparse=self.sparse_output, - fit_only=True) # these encoders won't have been fit yet - for var, var_ in zip(term.variables, term_.variables): - if var not in self.features_ and isinstance(var, Feature): - self.features_[var] = var_ - elif term not in self.column_info_: - # a tuple of features represents an interaction - if type(term) == type((1,)): - names = [] - column_map = {} - column_names = {} - idx = 0 - variables_ = [] - for var in term: - if var in self.features_: # already fit - var_ = self.features_[var] - else: - var_ = _clone_term(var, clone_memo) - if isinstance(var_, Column): - var_.fit_encoder(X) - else: - build_columns(self.column_info_, - X, - var_, - encoders=self.encoders_, - col_cache=tmp_cache, - fit=True, sparse=self.sparse_output, - fit_only=True) - # column identifiers are kept as is, Features and - # Columns are replaced by their fitted copies - variables_.append(var_ if isinstance(var, (Feature, Column)) else var) - var = var_ - cols, cur_names = build_columns(self.column_info_, - X_head, - var, - encoders=self.encoders_, - sparse=self.sparse_output) - column_map[var.name] = range(idx, idx + cols.shape[1]) - column_names[var.name] = cur_names - idx += cols.shape[1] - names.append(var.name) - encoder_ = Interaction(names, column_map, column_names) - self.features_[term] = Feature(tuple(variables_), ':'.join(n for n in names), encoder_) - elif isinstance(term, Column): - term_ = _clone_term(term, clone_memo) - self.features_[term] = Feature((term_,), term.name, None, pure_columns=True) - term_.fit_encoder(X) - else: - raise ValueError('each element in a term should be a Feature, Column or identify a column') - - # build the mapping of terms to columns and column names - - self.column_names_ = {} - self.column_map_ = {} - self.terms_ = [self.features_[t] for t in self.terms] - - idx = 0 - if self._include_intercept(): - self.column_map_['intercept'] = slice(0, 1) - idx += 1 # intercept will be first column - - for term, term_ in zip(self.terms, self.terms_): - term_df, term_names = build_columns(self.column_info_, - X_head, - term_, - encoders=self.encoders_, sparse=self.sparse_output) - self.column_names_[term] = term_names - self.column_map_[term] = slice(idx, idx + term_df.shape[1]) - idx += term_df.shape[1] - - return self - - def transform(self, X, y=None): - """ - Build design on X after fitting. - - Parameters - ---------- - X : array-like - - y : None - Ignored. This parameter exists only for compatibility with - :py:class:`sklearn.pipeline.Pipeline`. - """ - check_is_fitted(self) - X, to_native = _from_native(X, _term_columns(self.terms_)) - X = self._check_X(X, reset=False) - D = build_model(self.column_info_, - X, - self.terms_, - intercept=self._include_intercept(), - encoders=self.encoders_, - sparse=self.sparse_output, - order='C' if to_native is None else 'F') - return self._to_output(D, to_native) - - def _to_output(self, D, to_native): - # a model matrix for a non-pandas dataframe is returned in its - # backend, unless an output container was chosen with set_output - config = getattr(self, '_sklearn_output_config', {}).get('transform', 'default') - if to_native is None or config != 'default' or sp_sparse.issparse(D): - return D - return to_native(D) - - def get_feature_names_out(self, input_features=None): - """ - Names of the columns of the model matrix. - - Parameters - ---------- - input_features : None - Ignored. This parameter exists only for compatibility with - :py:class:`sklearn.pipeline.Pipeline`. - - Returns - ------- - names : np.ndarray of str - """ - check_is_fitted(self) - names = ['intercept'] if self._include_intercept() else [] - for term in self.terms: - names.extend(str(n) for n in self.column_names_[term]) - if not names: # build_model returns a 'zero' column - names = ['zero'] - return np.asarray(names, dtype=object) - - # ModelSpec specific methods - - @property - def names(self, help='Name for each term in model specification.'): - names = [] - if self._include_intercept(): - names = ['intercept'] - return names + [t.name for t in self.terms_] - - - def build_submodel(self, - X, - terms): - """ - Build design on X after fitting. - - Parameters - ---------- - X : array-like - X on which columns are evaluated. - - terms : [Feature] - Sequence of features. Elements of `self.terms` - are replaced by their fitted versions in `self.terms_`. - - Returns - ------- - D : array-like - Design matrix created with `terms` - """ - - check_is_fitted(self) - X, to_native = _from_native(X, _term_columns(terms)) - X = self._check_X(X, reset=False) - terms = [self.features_.get(t, t) for t in terms] - D = build_model(self.column_info_, - X, - terms, - intercept=self._include_intercept(), - encoders=self.encoders_, - sparse=self.sparse_output, - order='C' if to_native is None else 'F') - return self._to_output(D, to_native) - - def build_sequence(self, - X, - anova_type='sequential'): - """ - Build implied sequence of submodels - based on successively including more terms. - - Parameters - ---------- - X : array-like - X on which columns are evaluated. - - anova_type: str - One of "sequential" or "drop". - - Returns - ------- - - models : generator - Generator for sequence of models for ANOVA. - - """ - - check_is_fitted(self) - X, to_native = _from_native(X, _term_columns(self.terms_)) - X = self._check_X(X, reset=False) - - if anova_type not in ['sequential', 'drop']: - raise ValueError('anova_type must be one of ["sequential", "drop"]') - - if self.sparse_output: - blocks = [np.ones((X.shape[0], 1)) if self._include_intercept() else - np.zeros((X.shape[0], 1))] - for term_ in self.terms_: - blocks.append(build_columns(self.column_info_, - X, - term_, - encoders=self.encoders_, - fit=False, - sparse=True)[0]) - if anova_type == 'sequential': - return (_stack_sparse(blocks[:i], X.shape[0]) for i in range(1, len(blocks)+1)) - return (_stack_sparse([b for j, b in enumerate(blocks) if j != i], X.shape[0]) - for i in range(len(blocks))) - - col_cache = {} # avoid recomputing the same columns - - dfs = [] - - if self._include_intercept(): - df_int = pd.DataFrame({'intercept':np.ones(X.shape[0])}) - if isinstance(X, (pd.Series, pd.DataFrame)): - df_int.index = X.index - dfs.append(df_int) - else: - df_int = pd.DataFrame({'zero':np.zeros(X.shape[0])}) - if isinstance(X, (pd.Series, pd.DataFrame)): - df_int.index = X.index - dfs.append(df_int) - - for term_ in self.terms_: - term_df, _ = build_columns(self.column_info_, - X, - term_, - col_cache=col_cache, - encoders=self.encoders_, - fit=False) - if isinstance(X, (pd.Series, pd.DataFrame)): - term_df.index = X.index - # as in build_model, make boolean columns float - for i, col in enumerate(term_df.columns): - if term_df.iloc[:,i].dtype == bool: - term_df[col] = term_df.iloc[:,i].astype(float) - - dfs.append(term_df) - - if anova_type == 'sequential': - if isinstance(X, (pd.Series, pd.DataFrame)): - seq = (pd.concat(dfs[:i], axis=1) for i in range(1, len(dfs)+1)) - else: - seq = (np.column_stack(dfs[:i]) for i in range(1, len(dfs)+1)) - elif anova_type == 'drop': - if isinstance(X, (pd.Series, pd.DataFrame)): - seq = (pd.concat([dfs[j] for j in range(len(dfs)) if j != i], axis=1) for i in range(len(dfs))) - else: - seq = (np.column_stack([dfs[j] for j in range(len(dfs)) if j != i]) for i in range(len(dfs))) - else: - raise ValueError('anova_type must be one of ["sequential", "drop"]') - return (self._to_output(D, to_native) for D in seq) - -def _clone_term(term, memo): - """ - Copy a Feature or Column, replacing each encoder - (recursively) with an unfitted clone. - - `memo` maps `id()` of terms and encoders already copied - to their copies, so a Feature or encoder that appears - in several places is copied (and later fit) only once. - """ - - if not isinstance(term, (Feature, Column)): - return term # a column identifier - if id(term) in memo: - return memo[id(term)] - - encoder = term.encoder - if encoder is not None: - if id(encoder) not in memo: - memo[id(encoder)] = clone(encoder, safe=False) - encoder = memo[id(encoder)] - - if isinstance(term, Feature): - variables = tuple(_clone_term(v, memo) for v in term.variables) - term_ = term._replace(variables=variables, encoder=encoder) - else: - term_ = term._replace(encoder=encoder) - memo[id(term)] = term_ - return term_ - -def fit_encoder(encoders, var, X): - """ - Fit an encoder if not already registered - in `encoders`. - - Parameters - ---------- - - encoders : dict - Dictionary of encoders for each feature. - - var : Feature - Feature whose encoder will be fit. - - X : array-like - X on which encoder will be fit. - - """ - - if var.encoder not in encoders: - encoders[var] = var.encoder.fit(X) - -def build_columns(column_info, X, var, encoders={}, col_cache={}, fit=False, sparse=False, - fit_only=False, as_array=False): - """ - Build columns for a Feature from X. - - Parameters - ---------- - - column_info: dict - Dictionary with values specifying sets of columns to - be concatenated into a design matrix. - - X : array-like - X on which columns are evaluated. - - var : Feature - Feature whose columns will be built, typically a key in `column_info`. - - encoders : dict - Dict that stores encoder of each Feature. - - col_cache: dict - Dict where columns will be stored -- - if `var.name` in `col_cache` then just - returns those columns. - - fit : bool (optional) - If True, then try to fit encoder. - Will raise an error if encoder has already been fit. - - sparse : bool (optional) - If True, return the columns as a `scipy.sparse` matrix or - `np.ndarray` (not a `pd.DataFrame`), keeping categorical - encodings and their interactions sparse. - - fit_only : bool (optional) - With `fit`, fit the encoders of a Feature (and of the features - and columns it is built from) without evaluating its columns: - returns `(None, None)`. - - as_array : bool (optional) - If True, return the columns as a 2-d `np.ndarray` rather than - a `pd.DataFrame`. - - """ - - if var in column_info: - var = column_info[var] - - docache = DOCACHE and not sparse - if docache and joblib_hash([var, X]) in col_cache: - return col_cache[joblib_hash([var, X])] - - if isinstance(var, Column): - if docache: - if joblib_hash([var, X]) not in col_cache: - cols, names = var.get_columns(X, fit=fit) - col_cache[joblib_hash([var, X])] = cols, names - cols, name = col_cache[joblib_hash([var, X])] - else: - cols, names = var.get_columns(X, fit=fit, sparse=sparse) - elif isinstance(var, Feature): - if fit and fit_only and not var.encoder: - # nothing to fit here: only the variables need fitting - for v in var.variables: - if v in column_info: - continue - if isinstance(v, Feature): - build_columns(column_info, X, v, encoders=encoders, - col_cache=col_cache, fit=True, sparse=sparse, - fit_only=True) - elif isinstance(v, Column): - v.fit_encoder(X) - return None, None - cols = [] - names = [] - for v in var.variables: - cur, cur_names = build_columns(column_info, - X, - v, - encoders=encoders, - col_cache=col_cache, - fit=fit, - sparse=sparse, - as_array=True) - cols.append(cur) - names.extend(cur_names) - if any(sp_sparse.issparse(c) for c in cols): - cols = sp_sparse.hstack([c if sp_sparse.issparse(c) else - sp_sparse.csc_matrix(_as_2d(c)) for c in cols], - format='csc') - elif len(cols) == 1: # np.column_stack would copy it - cols = _as_2d(cols[0]) - else: - cols = np.column_stack(cols) - if len(names) != cols.shape[1]: - names = ['{0}[{1}]'.format(var.name, j) for j in range(cols.shape[1])] - if var.encoder: - if sp_sparse.issparse(cols) and isinstance(var.encoder, Interaction): - df_cols = cols # products of sparse columns stay sparse - else: - df_cols = pd.DataFrame(_dense(cols), - columns=names, - copy=False) - try: - check_is_fitted(var.encoder) - # ModelSpec.fit clones encoders, so one that is already fit - # was kept by clone(), e.g. a sklearn.frozen.FrozenEstimator: - # use it as is - if fit and var not in encoders: - encoders[var] = var.encoder - except NotFittedError as e: - if fit: - fit_encoder(encoders, - var, - df_cols) - # known issue with Pipeline - # https://github.com/scikit-learn/scikit-learn/issues/18648 - elif isinstance(var.encoder, Pipeline): - check_is_fitted(var.encoder, 'n_features_in_') - else: - raise(e) - except Exception as e: # was not the NotFitted - raise ValueError(e) - if fit_only: - return None, None - if var.use_transform: - cols = var.encoder.transform(df_cols) - else: - cols = var.encoder.predict(df_cols) - if not sparse: - cols = _dense(cols) - elif not sp_sparse.issparse(cols): - cols = np.asarray(cols) - if hasattr(var.encoder, 'columns_') and not var.override_encoder_colnames: - names = var.encoder.columns_ - else: - if cols.ndim > 1 and cols.shape[1] > 1: - names = ['{0}[{1}]'.format(var.name, j) for j in range(cols.shape[1])] - else: - names = [var.name] - - - else: - raise ValueError('expecting either a column or a Feature') - if sparse: - return (cols if sp_sparse.issparse(cols) else _as_2d(cols)), names - if as_array: - return _as_2d(_dense(cols)), names - val = pd.DataFrame(_dense(cols), columns=names, copy=False) - - if isinstance(X, (pd.DataFrame, pd.Series)): - val.index = X.index - - if docache: - col_cache[joblib_hash([var.name, X])] = (val, names) - return val, names - -def _fill_model(column_info, X, terms, intercept, encoders, order='C'): - # the dense model matrix, as float: its columns are allocated once - # (their number found from the first rows of X) and each term's - # block is copied in and freed, so the peak memory is the model - # matrix and its largest block. order='F' stores each column - # contiguously, so that e.g. polars can take them without a copy - X_head = X.iloc[:2] if isinstance(X, pd.DataFrame) else X[:2] - names, widths = (['intercept'], [1]) if intercept else ([], []) - for term_ in terms: - head, head_names = build_columns(column_info, X_head, term_, - encoders=encoders, as_array=True) - names.extend(head_names) - widths.append(head.shape[1]) - D = np.empty((X.shape[0], sum(widths)), order=order) - start = 0 - if intercept: - D[:, 0] = 1 - start = 1 - for term_, width in zip(terms, widths[1:] if intercept else widths): - block = build_columns(column_info, X, term_, encoders=encoders, - as_array=True)[0] - D[:, start:start + width] = block - start += width - del block - if isinstance(X, (pd.DataFrame, pd.Series)): - return pd.DataFrame(D, columns=names, index=X.index, copy=False) - return D - -def _dense(cols): - return cols.toarray() if sp_sparse.issparse(cols) else np.asarray(cols) - -def _as_2d(cols): - cols = np.asarray(cols) - return cols.reshape((cols.shape[0], -1)) - -def _stack_sparse(blocks, n): - # blocks (sparse or dense) as one CSR matrix of floats - if not blocks: - blocks = [np.zeros((n, 1))] - return sp_sparse.hstack([b if sp_sparse.issparse(b) else - sp_sparse.csr_matrix(_as_2d(b).astype(float)) - for b in blocks], format='csr', dtype=float) - -def build_model(column_info, - X, - terms, - intercept=True, - encoders={}, - sparse=False, - order='C'): - - """ - Construct design matrix on a - sequence of terms and X after - fitting. - - Parameters - ---------- - column_info: dict - Dictionary with values specifying sets of columns to - be concatenated into a design matrix. - - X : array-like - X on which columns are evaluated. - - terms : [Feature] - Sequence of features - - encoders : dict - Dict that stores encoder of each Feature. - - sparse : bool - If True, return a `scipy.sparse.csr_matrix`. - - order : 'C' or 'F' - The memory layout of a dense model matrix. - - Returns - ------- - df : np.ndarray, pd.DataFrame or scipy.sparse.csr_matrix - Design matrix. - """ - - if sparse: - blocks = [np.ones((X.shape[0], 1))] if intercept else [] - for term_ in terms: - blocks.append(build_columns(column_info, - X, - term_, - encoders=encoders, - fit=False, - sparse=True)[0]) - return _stack_sparse(blocks, X.shape[0]) - - if terms or intercept: - return _fill_model(column_info, X, terms, intercept, encoders, order=order) - - dfs = [] - - col_cache = {} # avoid recomputing the same columns - - if intercept: - df = pd.DataFrame({'intercept':np.ones(X.shape[0])}) - if isinstance(X, (pd.Series, pd.DataFrame)): - df.index = X.index - dfs.append(df) - - for term_ in terms: - term_df = build_columns(column_info, - X, - term_, - col_cache=col_cache, - encoders=encoders, - fit=False)[0] - dfs.append(term_df) - - if len(dfs): - if isinstance(X, (pd.Series, pd.DataFrame)): - df = pd.concat(dfs, axis='columns') - df.index = X.index - else: - return np.column_stack(dfs).astype(float) - else: # return a 0 design - zero = np.zeros(X.shape[0]) - if isinstance(X, (pd.Series, pd.DataFrame)): - df = pd.DataFrame({'zero': zero}) - df.index = X.index - else: - return zero - - # if we reach here, we will be returning a DataFrame - # make sure all columns are floats - - for i, col in enumerate(df.columns): - if df.iloc[:,i].dtype == bool: - df[col] = df.iloc[:,i].astype(float) - return df - -def derived_feature(variables, encoder=None, name=None, use_transform=True): - """ - Create a Feature, optionally - applying an encoder to the stacked columns. - - Parameters - ---------- - - variables : [column identifier, Column, Feature] - Variables to apply transform to. Could be - column identifiers or variables: all columns - will be stacked before encoding. - - name : str (optional) - Defaults to `str(encoder)`. - - encoder : transform-like (optional) - Transform obeying sklearn fit/transform convention. - - Returns - ------- - - var : Feature - """ - - if name is None: - name = str(encoder) - var = Feature(tuple([v for v in variables]), - name, - encoder, - use_transform=use_transform, - override_encoder_colnames=True) - return var - -def contrast(col, - method='drop', - drop_level=None): - """ - Create encoding of categorical feature. - - Parameters - ---------- - - col: column identifier - - method: 'drop', 'sum', None or callable - - drop_level: level identifier - - Returns - ------- - - var : Feature - - """ - - if isinstance(col, Column): - name = col.name - col = col.idx - else: - name = str(col) - encoder = Contrast(method, - drop_level=drop_level) - return Column(col, - name, - is_categorical=True, - encoder=encoder) - -def ordinal(col, name=None, levels=None): - """ - Create ordinal encoding of categorical feature: a single - column holding the position of each entry's level (see `Ordinal`). - - Parameters - ---------- - - col: column identifier - - name: str (optional) - Defaults to `'ordinal(col)'`. - - levels: sequence (optional) - The levels in order. Needed unless `col` is an ordered - categorical, or numeric. - - Returns - ------- - - var : Column - - """ - - if isinstance(col, Column): - colname = col.name - col = col.idx - else: - colname = str(col) - if name is None: - name = f'ordinal({colname})' - return Column(col, - name, - is_categorical=True, - is_ordinal=True, - columns=(name,), - encoder=Ordinal(levels=levels)) - -def poly(col, - degree=1, - intercept=False, - raw=False, - name=None): - - """ - Create a polynomial Feature - for a given column. - - Additional `args` and `kwargs` - are passed to `Poly`. - - Parameters - ---------- - - col : column identifier or Column - Column to transform. - - degree : int, default=1 - Degree of polynomial. - - intercept : bool, default=False - Include a column for intercept? - - raw : bool, default=False - If False, perform a QR decomposition on the resulting - matrix of powers of centered and / or scaled features. - - name : str (optional) - Defaults to one derived from col. - - Returns - ------- - - var : Feature - """ - shortname, klass = 'poly', Poly - spec = _spec('poly', (col,), degree=degree, intercept=intercept, raw=raw, name=name) - encoder = klass(degree=degree, - raw=raw, - intercept=intercept) - if name is None: - if isinstance(col, Column): - name = col.name - else: - name = str(col) - - kwargs = {} - if intercept: - kwargs['intercept'] = True - if raw: - kwargs['raw'] = True - - _args = _argstring(degree=degree, - **kwargs) - if _args: - name = ', '.join([name, _args]) - - name = f'{shortname}({name})' - - return derived_feature([col], - name=name, - encoder=encoder)._replace(spec=spec) - -def ns(col, intercept=False, name=None, **spline_args): - """ - Create a natural spline Feature - for a given column. - - Additional *spline_args* - are passed to :py:class:`NaturalSpline` along with *intercept*. - - Parameters - ---------- - - col : column identifier or Column - - intercept : bool - Include an intercept column. - - name : str (optional) - Defaults to one derived from col. - - Returns - ------- - - var : Feature - - """ - shortname, klass = 'ns', NaturalSpline - spec = _spec('ns', (col,), intercept=intercept, name=name, **spline_args) - if name is None: - if isinstance(col, Column): - name = col.name - else: - name = str(col) - - _args = _argstring(**spline_args) - if _args: - name = ', '.join([name, _args]) - - name = f'{shortname}({name})' - encoder = klass(intercept=intercept, - **spline_args) - return derived_feature([col], - name=name, - encoder=encoder)._replace(spec=spec) - -def bs(col, intercept=False, name=None, **spline_args): - """ - Create a B-spline Feature - for a given column. - - Additional args and *spline_args* - are passed to :py:class:`ISLP.transforms.BSpline` - along with *intercept*. - - Parameters - ---------- - - col : column identifier or Column - - intercept : bool - Include an intercept column. - - name : str (optional) - Defaults to one derived from col. - - Returns - ------- - - var : Feature - - """ - shortname, klass = 'bs', BSpline - spec = _spec('bs', (col,), intercept=intercept, name=name, **spline_args) - if name is None: - if isinstance(col, Column): - name = col.name - else: - name = str(col) - - _args = _argstring(**spline_args) - if _args: - name = ', '.join([name, _args]) - - name = f'{shortname}({name})' - encoder = klass(intercept=intercept, - **spline_args) - return derived_feature([col], - name=name, - encoder=encoder)._replace(spec=spec) - -def pca(variables, name, scale=False, **pca_args): - """ - Create PCA encoding of features - from a sequence of variables. - - Additional args and *pca_args* - are passed to :py:class:`ISLP.transforms.PCA`. - - Parameters - ---------- - - variables : [column identifier, Column or Feature] - Sequence whose columns will be encoded by PCA. - - Returns - ------- - - var : Feature - - """ - shortname, klass = 'pca', PCA - spec = _spec('pca', (variables, name), scale=scale, **pca_args) - encoder = klass(**pca_args) - if scale: - scaler = StandardScaler(with_mean=True, - with_std=True) - encoder = make_pipeline(scaler, encoder) - - _args = _argstring(**pca_args) - - if _args: - name = ', '.join([name, _args]) - - return derived_feature(variables, - name=f'{shortname}({name})', - encoder=encoder)._replace(spec=spec) - - -def _argstring(*args, **kwargs): - _args = ', '.join([str(a) for a in args]) - _kwargs = ', '.join([f'{k}={v}' for k, v in kwargs.items()]) - - if args and kwargs: - return ', '.join([_args, _kwargs]) - elif args: - return _args - elif kwargs: - return _kwargs - else: - return '' - -#### parameters of the terms of a ModelSpec, e.g. for GridSearchCV - -def _freeze(v): - # a hashable version of a parameter value, so that Features stay hashable - if isinstance(v, np.ndarray): - return tuple(_freeze(x) for x in v.tolist()) - if isinstance(v, (list, tuple)) and not isinstance(v, (Feature, Column)): - return tuple(_freeze(x) for x in v) - return v - -def _spec(helper, args, **kwargs): - # how `helper` was called; a name is kept only if one was given - if kwargs.get('name') is None: - kwargs.pop('name', None) - return (helper, tuple(_freeze(a) for a in args), - tuple(sorted((k, _freeze(v)) for k, v in kwargs.items()))) - -def _rebuild(feature, **changes): - # the Feature made again by its helper, with some parameters changed - helper, args, kwargs = feature.spec - kwargs = dict(kwargs) - kwargs.update(changes) - if helper == 'pca': - args = (list(args[0]),) + tuple(args[1:]) - return {'poly': poly, 'ns': ns, 'bs': bs, 'pca': pca}[helper](*args, **kwargs) - -def _encoder_params(encoder): - if encoder is None or not hasattr(encoder, 'get_params'): - return {} - if isinstance(encoder, Pipeline): # pca(..., scale=True): the PCA's parameters - encoder = encoder.steps[-1][1] - return encoder.get_params(deep=False) - -def _term_params(term): - """ - The parameters of a term that `ModelSpec.set_params` can change, - e.g. {'df': 4, 'intercept': False, ...} for ns('x', df=4). - """ - if isinstance(term, Feature): - params = _encoder_params(term.encoder) - if term.spec is not None: - params.update((k, v) for k, v in term.spec[2] if k != 'name') - return params - if isinstance(term, Column): - return _encoder_params(term.encoder) - if isinstance(term, tuple): # an interaction: the parameters of its parts - return {f'{j}__{k}': v for j, part in enumerate(term) - for k, v in _term_params(part).items()} - return {} - -def _set_term_param(term, path, value): - """ - A copy of `term` with the parameter at `path` (e.g. ['df'], or - ['0', 'degree'] for a part of an interaction) set to `value`, and the - (old, new) pair of the Feature or Column that was replaced. - """ - if isinstance(term, tuple) and not isinstance(term, (Feature, Column)): - if not path or not path[0].isdigit() or int(path[0]) >= len(term): - raise ValueError(f'expecting the position of a part of the interaction {term!r}, ' - f'got {"__".join(path)!r}') - j = int(path[0]) - part, replaced = _set_term_param(term[j], path[1:], value) - return term[:j] + (part,) + term[j+1:], replaced - if not isinstance(term, (Feature, Column)): - raise ValueError(f'the term {term!r} has no parameters to set') - valid = _term_params(term) - if len(path) != 1 or path[0] not in valid: - raise ValueError(f'invalid parameter {"__".join(path)!r} for the term {term.name!r}; ' - f'valid parameters are {sorted(valid)}') - param = path[0] - if isinstance(term, Feature) and term.spec is not None: - new = _rebuild(term, **{param: value}) - else: - encoder = term.encoder - if isinstance(encoder, Pipeline): - encoder = clone(encoder).set_params(**{f'{encoder.steps[-1][0]}__{param}': value}) - else: - encoder = clone(encoder).set_params(**{param: value}) - new = term._replace(encoder=encoder) - return new, (term, new) +import sys -def _substitute(term, old, new): - # `term` with every occurrence of the object `old` replaced by `new` - if term is old: - return new - if isinstance(term, Feature): - variables = tuple(_substitute(v, old, new) for v in term.variables) - if any(a is not b for a, b in zip(variables, term.variables)): - return term._replace(variables=variables) - return term - if isinstance(term, tuple) and not isinstance(term, Column): - parts = tuple(_substitute(v, old, new) for v in term) - return parts if any(a is not b for a, b in zip(parts, term)) else term - return term +from modelmatrix import model_spec as _module -def _term_position(terms, key): - # the position in `terms` of the term named by `key`: a position, or a name - if key.isdigit(): - if int(key) >= len(terms): - raise ValueError(f'there is no term at position {key}: there are {len(terms)} terms') - return int(key) - found = [i for i, t in enumerate(terms) if getattr(t, 'name', None) == key] - if len(found) != 1: - raise ValueError(f'expecting a position in terms, or the name of exactly one term, ' - f'got {key!r} ({len(found)} terms have that name)') - return found[0] +sys.modules[__name__] = _module diff --git a/ISLP/models/strategy.py b/ISLP/models/strategy.py index 3740865..9444bab 100644 --- a/ISLP/models/strategy.py +++ b/ISLP/models/strategy.py @@ -22,10 +22,6 @@ from sklearn.utils import check_random_state from sklearn.utils.validation import check_is_fitted -from .columns import (_get_column_info, - Column, - _categorical_from_df, - _check_categories) class Strategy(NamedTuple): @@ -85,7 +81,9 @@ def __init__(self, """ self.model_spec = model_spec - nterms = len(self.model_spec.terms) + check_is_fitted(self.model_spec) + # the fitted terms: a wildcard stands for several + nterms = len(self.model_spec.terms_) if (not isinstance(max_terms, int) or (max_terms > nterms or max_terms < 0)): @@ -107,26 +105,12 @@ def __init__(self, self._have_already_run = False if lower_terms: - lower_terms_ = [] - for term in lower_terms: - mm_terms = list(self.model_spec.terms) - if term in mm_terms: - idx = mm_terms.index(term) - term = self.model_spec.terms_[idx] - lower_terms_.append(term) - self.lower_terms = set(lower_terms_) + self.lower_terms = set(_fitted_terms(self.model_spec, lower_terms)) else: self.lower_terms = set([]) if upper_terms: - upper_terms_ = [] - for term in upper_terms: - mm_terms = list(self.model_spec.terms) - if term in mm_terms: - idx = mm_terms.index(term) - term = self.model_spec.terms_[idx] - upper_terms_.append(term) - self.upper_terms = set(upper_terms_) + self.upper_terms = set(_fitted_terms(self.model_spec, upper_terms)) else: self.upper_terms = set(self.model_spec.terms_) @@ -394,14 +378,7 @@ def first_peak(model_spec, # pick an initial state if initial_terms is not None and initial_terms != []: - initial_terms_ = [] - mm_terms = list(model_spec.terms) - for term in initial_terms: - if term in mm_terms: - idx = mm_terms.index(term) - term = model_spec.terms_[idx] - initial_terms_.append(term) - initial_state = tuple(initial_terms_) + initial_state = tuple(_fitted_terms(model_spec, initial_terms)) else: # backward search starts from the full model initial_state = tuple(model_spec.terms_) if direction == 'backward' else () @@ -456,7 +433,7 @@ def fixed_steps(model_spec, """ - n_terms = n_steps if direction == 'forward' else len(list(model_spec.terms)) - n_steps + n_terms = n_steps if direction == 'forward' else len(model_spec.terms_) - n_steps step = Stepwise(model_spec, direction=direction, min_terms=n_terms, @@ -468,14 +445,7 @@ def fixed_steps(model_spec, # pick an initial state if initial_terms is not None and initial_terms != []: - initial_terms_ = [] - mm_terms = list(model_spec.terms) - for term in initial_terms: - if term in mm_terms: - idx = mm_terms.index(term) - term = model_spec.terms_[idx] - initial_terms_.append(term) - initial_state = tuple(initial_terms_) + initial_state = tuple(_fitted_terms(model_spec, initial_terms)) else: initial_state = () if direction == 'forward' else list(model_spec.terms_) @@ -640,8 +610,20 @@ def fixed_steps(n_steps, # private functions -def _build_submodel(column_info, X, cols): - return np.column_stack([column_info[col].get_columns(X, fit=True)[0] for col in cols]) +def _fitted_terms(model_spec, terms): + """ + The fitted versions of `terms`, given as in the terms of a fitted + `model_spec` (or as those a wildcard stands for, in its + `expanded_terms_`): elements of its `terms_`. Other terms are + returned as they are. + """ + fitted = [] + for term in terms: + try: + fitted.append(model_spec.features_.get(term, term)) + except TypeError: # not hashable: not a term of the ModelSpec + fitted.append(term) + return fitted def _postprocess_fixed_steps(n_steps, results): diff --git a/ISLP/transforms.py b/ISLP/transforms.py index 4c8cf60..9664eae 100644 --- a/ISLP/transforms.py +++ b/ISLP/transforms.py @@ -1,695 +1,10 @@ """ -Transformers -============ - -This module defines some sklearn transformers useful for defining -flexible regression models for single features, as well as interactions -between sets of columns. - - - Poly: for orthogonalized polynomial regression - - - Interaction: computing pairwise product for interactions - - - BSpline: arbitrary degree B-splines - - - NaturalSpline: natural cubic splines +Moved to the modelmatrix package, as `modelmatrix.transforms`: this module +is `modelmatrix.transforms` itself, so that imports from it keep working. """ -from itertools import product -import numpy as np, pandas as pd - -from sklearn.base import TransformerMixin, BaseEstimator -from sklearn.utils.validation import check_is_fitted -from scipy.interpolate import splev -from scipy import sparse as sp_sparse - -def _single_column(X): - """ - A single feature as a 1-d array of floats (a copy), from a 1-d - array, a column (e.g. a `pd.Series` or a 1-column `pd.DataFrame`) or - a single value. A single row stays one value (np.squeeze made it 0-d). - """ - X = np.array(X, dtype=float) - if X.ndim == 2 and X.shape[1] == 1: - X = X[:, 0] - elif X.ndim == 0: - X = X.reshape(1) - if X.ndim != 1: - raise ValueError('expecting a single column feature') - return X - -class Poly(TransformerMixin, BaseEstimator): - - ''' - - Parameters - ---------- - - degree : int, default=1 - Degree of polynomial. - - intercept : bool, default=False - Include a column for intercept? - - raw : bool, default=False - If False, perform a QR decomposition on the resulting - matrix of powers of centered and / or scaled features. - ''' - - def __init__(self, - degree=1, - intercept=False, - raw=False): - - self.degree = degree - self.raw = raw - self.intercept = intercept - - def fit(self, - X, - y=None): - - """ - Construct parameters for orthogonal - polynomials in the feature X. - - Parameters - ---------- - X : array-like of shape (n_samples,) - Features used in fitting `svm`. Assumed to have at least 2 columns. - - y : default=None - Ignored. This parameter exists only for compatibility with - :class:`~sklearn.pipeline.Pipeline`. - - """ - - X_orig = X - X = _single_column(X) - n = X.shape[0] - - self.mean_ = X.mean() - - if not self.raw: - powX = np.power.outer(X - self.mean_, np.arange(0, self.degree+1)) - # Following R's poly construction - Q, R = np.linalg.qr(powX) - Z = Q * np.diag(R)[None,:] - self.norm2_ = (Z**2).sum(0) - self.alpha_ = ((X[:,None] * Z**2).sum(0) / self.norm2_)[:self.degree] - self.norm2_ = np.hstack([1, self.norm2_]) - - # for pandas - - self.columns_ = range(self.degree+1) - if not self.intercept: - self.columns_ = self.columns_[:-1] - - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - if isinstance(X_orig, pd.Series): - name = X_orig.name - else: - name = X_orig.columns[0] - self.columns_ = ['{0}[{1}]'.format(self, d) - for d in self.columns_] - - return self - - def transform(self, X): - - """ - Construct parameters for orthogonal - polynomials in the feature X. - - Parameters - ---------- - X : array-like - X on which features will be evaluated. - - Returns - ------- - XP : np.ndarray - Evaluated polynomial features. - """ - check_is_fitted(self) - - X_orig = X - - X = _single_column(X) - n = X.shape[0] - - if not self.raw: - Z = np.ones((n, self.degree+1)) - Z[:,1] = X - self.alpha_[0] - - if self.degree > 1: - for i in range(1, self.degree): - Z[:,i+1] = ((X - self.alpha_[i]) * Z[:,i] - - self.norm2_[i+1] / self.norm2_[i] * Z[:,i-1]) - Z /= np.sqrt(self.norm2_[1:]) - powX = Z - else: - powX = np.power.outer(X, np.arange(0, self.degree+1)) - - if not self.intercept: - powX = powX[:,1:] - - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - df = pd.DataFrame(powX, - columns=self.columns_, - copy=False) - df.index = X_orig.index - return df - else: - return powX - -#### Interaction transform - -class Interaction(TransformerMixin, BaseEstimator): - - ''' - - Form the tensor product interaction - from a group of columns. - - Parameters - ---------- - - variables : sequence - Variables in the interactions. - - columns : dict - Mapping from variable names to columns. - - column_names : dict - Mapping from variable names to lists of column names. - - ''' - - def __init__(self, - variables, - columns, - column_names): - - self.variables = variables - self.columns = columns - self.column_names = column_names - self.column_names_ = {} - self.columns_ = [] - - variable_names = [] - for variable in self.variables: - cols = self.columns[variable] - col_names = ['{0}[{1}]'.format(variable, i) for i in range(len(self.columns[variable]))] - if variable in column_names: - col_names = [str(c) for c in column_names[variable]] - if len(cols) > 1: - variable_names.append(col_names) - else: - variable_names.append(['{0}'.format(variable)]) - - for names in product(*variable_names): - self.columns_.append(':'.join(names)) - - def fit(self, - X, - y=None): - - """ - Nothing to be computed for the fit. - - Parameters - ---------- - X : array-like - - y : default=None - Ignored. This parameter exists only for compatibility with - :class:`~sklearn.pipeline.Pipeline`. - - """ - - return self - - def transform(self, X): - - """ - Construct columns representing interactions - of relevant variables. - - Parameters - ---------- - X : array-like - X on which features will be evaluated. - - Returns - ------- - XI : np.ndarray - Evaluated interaction features. - """ - check_is_fitted(self) - - if sp_sparse.issparse(X): # products of sparse columns stay sparse - X = sp_sparse.csc_matrix(X) - R = None - for variable in self.variables: - V = X[:, list(self.columns[variable])] - if R is None: - R = V - continue - # column i*q + j of the product is R[:,i] * V[:,j], as for - # the dense columns below: scale the rows of R by each column - # of V, then reorder - r, q = R.shape[1], V.shape[1] - M = sp_sparse.hstack([sp_sparse.diags(V[:, [j]].toarray().ravel()) @ R - for j in range(q)], format='csc') - R = M[:, (np.arange(q)[None, :] * r + np.arange(r)[:, None]).ravel()] - return sp_sparse.csr_matrix(R) - - X_orig = X - X = np.asarray(X) - - # each column of the product is filled in place - idx = list(product(*[self.columns[v] for v in self.variables])) - value = np.empty((X.shape[0], len(idx))) - for k, cols in enumerate(idx): - value[:, k] = X[:, cols[0]] - for j in cols[1:]: - value[:, k] *= X[:, j] - - df = pd.DataFrame(value, - columns=self.columns_, - copy=False) - if isinstance(X_orig, (pd.DataFrame, pd.Series)): - df.index = X_orig.index - - return df - -#### Spline specific code - -def _onehot(p, j): - v = np.zeros(p) - v[j] = 1 - return v - -def _splevf(x, - tk, - der=0, - ext=0): - """ - Full B-spline matrix at x - """ - x = np.asarray(x) - knots, degree = tk - nbasis = len(knots) - (degree + 1) - tcks = [(knots, _onehot(nbasis, j), degree) for - j in range(nbasis)] - return np.column_stack([splev(x, tck, ext=ext, der=der) - for tck in tcks]) - -def _splev_taylor(x, - basept, - tk, - order): - """ - Taylor expansion of B-spline at x around basept - """ - x = np.asarray(x) - derivs = np.array([_splevf([basept], - tk, - der=o, - ext=0).reshape(-1) - for o in range(order+1)]) - polys = np.power.outer(x-basept, - np.arange(order+1)) - fact = np.concatenate([[1], np.cumprod(np.arange(1, order+1))]) - polys /= fact[None,:] - - return polys.dot(derivs) - -class BSpline(TransformerMixin, BaseEstimator): - - ''' - - Parameters - ---------- - - degree : int, default=3 - Degree of polynomial. - - intercept : bool, default=False - If False, a column of basis is dropped so that by - adding an intercept column design stays full rank. - - lower_bound : float, default=None - Lower boundary not. Will be set to minimal value if not supplied. - - upper_bound : float, default=None - Upper boundary not. Will be set to maximal value if not supplied. - - internal_knots : array-like (optional) - Optional internal knots of B-spline. Will be set to - appropriate quantiles based on `df`. - - df : int, default=None - Degrees of freedom for spline. Defaults to `degree + intercept`. - - ext : int - How B-splines are to be extended beyond the boundary using - `scipy.interpolate.splev`. - - ''' - - def __init__(self, - degree=3, - intercept=False, - lower_bound=None, - upper_bound=None, - internal_knots=None, - df=None, - ext=0): - - self.degree = degree - self.lower_bound = lower_bound - self.upper_bound = upper_bound - self.internal_knots = internal_knots - self.df = df - self.ext = ext - self.intercept = intercept - - def fit(self, - X, - y=None): - - """ - Compute knots for B-spline representation. - - Parameters - ---------- - X : array-like - Single feature on which B-spline will be evaluated. - - y : None - Ignored. This parameter exists only for compatibility with - :class:`~sklearn.pipeline.Pipeline`. - - """ - - X_orig = X - X = _single_column(X) - n = X.shape[0] - - order = self.degree + 1 - - lower_bound = self.lower_bound - if lower_bound is None: - lower_bound = X.min() - - upper_bound = self.upper_bound - if upper_bound is None: - upper_bound = X.max() - - if self.df is not None: - if self.df < order - 1 + self.intercept: - raise ValueError('df must be greater than or equal to %d' % (order - 1 + self.intercept)) - ninternal = self.df - (order - 1 + self.intercept) - percs = 100*np.linspace(0, 1, ninternal+2)[1:-1] - internal_knots = np.percentile(X, percs) - if self.internal_knots is not None: - raise ValueError('only one of df or internal_knots should be specified') - else: - internal_knots = np.asarray(sorted(self.internal_knots)) - if self.internal_knots is None: - raise ValueError('if df not specified then need internal_knots') - - if lower_bound >= upper_bound: - raise ValueError('lower_bound must be smaller than upper_bound') - - self.internal_knots_ = internal_knots - - self.knots_ = np.sort(np.concatenate([[lower_bound]*order, - [upper_bound]*order, - internal_knots])) - if self.knots_[0] < lower_bound: - raise ValueError('internal_knots should be greater than our equal to lower_bound') - if self.knots_[-1] > upper_bound: - raise ValueError('internal_knots should be less than our equal to upper_bound') - - self.boundary_knots_ = [lower_bound, upper_bound] - - # for pandas - - self.nbasis_ = len(self.knots_) - (self.degree + 1) - self.columns_ = range(self.nbasis_) - - if not self.intercept: - self.columns_ = self.columns_[:-1] - - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - if isinstance(X_orig, pd.Series): - name = X_orig.name - else: - name = X_orig.columns[0] # a pd.DataFrame - self.columns_ = ['{0}[{1}]'.format(self, d) - for d in self.columns_] - return self - - def transform(self, X): - - """ - Construct design for B-splines - using features X. - - Parameters - ---------- - X : array-like - X on which splines will be evaluated. - - Returns - ------- - XS : np.ndarray - Evaluated splines. - """ - check_is_fitted(self) - - X_orig = X - X = _single_column(X) - n = X.shape[0] - - value = _splevf(X, - (self.knots_, self.degree), - der=0, - ext=self.ext) - if not self.intercept: - value = value[:,1:] - columns_ = self.columns_ - - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - df = pd.DataFrame(value, - columns=columns_, - copy=False) - df.index = X_orig.index - return df - else: - return value - -class NaturalSpline(TransformerMixin, BaseEstimator): - - ''' - - Natural cubic spline. - - Parameters - ---------- - - intercept : bool, default=False - If False, a column of basis is dropped so that by - adding an intercept column design stays full rank. - - lower_bound : float, default=None - Lower boundary not. Will be set to minimal value if not supplied. - - upper_bound : float, default=None - Upper boundary not. Will be set to maximal value if not supplied. - - internal_knots : array-like (optional) - Optional internal knots of B-spline. Will be set to - appropriate quantiles based on `df`. - - df : int, default=None - Degrees of freedom for spline. Defaults to `3 + intercept`. - - ext : int, default=0 - How B-splines are to be extended beyond the boundary using - `scipy.interpolate.splev`. - - ''' - - def __init__(self, - intercept=False, - lower_bound=None, - upper_bound=None, - internal_knots=None, - df=None, - ext=0): - - self.lower_bound = lower_bound - self.upper_bound = upper_bound - self.internal_knots = internal_knots - self.df = df - self.intercept = intercept - self.ext = ext - - def fit(self, - X, - y=None): - - """ - Compute knots for natural spline representation. - - Parameters - ---------- - X : array-like - - Single feature on which B-spline will be evaluated. - - y : None - Ignored. This parameter exists only for compatibility with - :class:`~sklearn.pipeline.Pipeline`. - - """ - - X_orig = X - X = _single_column(X) - n = X.shape[0] - - order = 4 - - lower_bound = self.lower_bound - if lower_bound is None: - lower_bound = X.min() - - upper_bound = self.upper_bound - if upper_bound is None: - upper_bound = X.max() - - if self.df is not None: - if self.df < order - 1 + self.intercept - 2: - raise ValueError('df must be greater than or equal to %d' % (order - 1 + self.intercept - 2)) - ninternal = self.df - (order - 1 + self.intercept - 2) # -2 for constraints - percs = 100*np.linspace(0, 1, ninternal+2)[1:-1] - internal_knots = np.percentile(X, percs) - if self.internal_knots is not None: - raise ValueError('only one of df or internal_knots should be specified') - else: - internal_knots = np.asarray(sorted(self.internal_knots)) - if self.internal_knots is None: - raise ValueError('if df not specified then need internal_knots') - - if lower_bound >= upper_bound: - raise ValueError('lower_bound must be smaller than upper_bound') - - self.internal_knots_ = internal_knots - self.knots_ = np.sort(np.concatenate([[lower_bound]*order, - [upper_bound]*order, - internal_knots])) - if self.knots_[0] < lower_bound: - raise ValueError('internal_knots should be greater than our equal to lower_bound') - if self.knots_[-1] > upper_bound: - raise ValueError('internal_knots should be less than our equal to upper_bound') - - self.boundary_knots_ = [lower_bound, upper_bound] - self.nbasis_ = self.knots_.shape[0] - order - - # now enforce constraint that 2nd derivative at two boundary knots is 0 - # this depends on the X that is passed to fit! - - boundary_vals = _splevf(self.boundary_knots_, - (self.knots_, 3), - der=2, - ext=0) - if not self.intercept: - boundary_vals = boundary_vals[:,1:] - Q_, R_ = np.linalg.qr(boundary_vals.T, - mode='complete') - self.Qproj_ = Q_[:,2:] - - # for pandas - - self.columns_ = range(self.nbasis_-2) # two constraints - - if not self.intercept: - self.columns_ = self.columns_[:-1] - - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - if isinstance(X_orig, pd.Series): - name = X_orig.name - else: - name = X_orig.columns[0] # a pd.DataFrame - self.columns_ = ['{0}[{1}]'.format(self, d) - for d in self.columns_] - return self - - def transform(self, X): - - """ - Construct design for natural cubic splines - for features X. - - Parameters - ---------- - X : array-like - X on which natural cubic splines will be evaluated. - - Returns - ------- - XN : np.ndarray - Evaluated natural cubic spline features. - """ - - check_is_fitted(self) - - lower_bound, upper_bound = self.boundary_knots_ - - X_orig = X - X = _single_column(X) - n = X.shape[0] - - value = np.empty((X.shape[0], self.nbasis_)) - inside = (X >= lower_bound) * (X <= upper_bound) - if np.any(inside): - value[inside] = _splevf(X[inside], - (self.knots_, 3), - der=0, - ext=self.ext) - - to_the_right = (X > upper_bound) - if np.any(to_the_right): - value[to_the_right] = _splev_taylor(X[to_the_right], - upper_bound, - (self.knots_, 3), - order=1) - - to_the_left = (X < lower_bound) - if np.any(to_the_left): - value[to_the_left] = _splev_taylor(X[to_the_left], - lower_bound, - (self.knots_, 3), - order=1) - - # possibly drop column for intercept - - if not self.intercept: - value = value[:,1:] - - # map onto the natural splines +import sys - value = value.dot(self.Qproj_) - if isinstance(X_orig, (pd.Series, pd.DataFrame)): - df = pd.DataFrame(value, - columns=self.columns_, - copy=False) - df.index = X_orig.index - return df - else: - return value +from modelmatrix import transforms as _module +sys.modules[__name__] = _module diff --git a/docs/jupyterbook/models/anova.ipynb b/docs/jupyterbook/models/anova.ipynb deleted file mode 100644 index bf77f73..0000000 --- a/docs/jupyterbook/models/anova.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "ee33d364", - "metadata": {}, - "source": [ - "# ANOVA using `ModelSpec`\n", - "\n", - "\n", - "In this lab we illustrate how to run create specific ANOVA analyses\n", - "using `ModelSpec`." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "4c70fbaa", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from statsmodels.api import OLS\n", - "from statsmodels.stats.anova import anova_lm\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec,\n", - " derived_feature,\n", - " summarize)" - ] - }, - { - "cell_type": "markdown", - "id": "333a49cf", - "metadata": {}, - "source": [ - "We will carry out two simple ANOVA analyses of the `Hitters` data.\n", - "We wish to predict a baseball player’s `Salary` on the\n", - "basis of various statistics associated with performance in the\n", - "previous year." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8a708215", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "59" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Hitters = load_data('Hitters')\n", - "np.isnan(Hitters['Salary']).sum()" - ] - }, - { - "cell_type": "markdown", - "id": "dad5e991", - "metadata": {}, - "source": [ - " \n", - " We see that `Salary` is missing for 59 players. The\n", - "`dropna()` method of data frames removes all of the rows that have missing\n", - "values in any variable (by default --- see `Hitters.dropna?`)." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ac7086a5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years', 'CAtBat',\n", - " 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'League', 'Division',\n", - " 'PutOuts', 'Assists', 'Errors', 'Salary', 'NewLeague'],\n", - " dtype='object')" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Hitters = Hitters.dropna()\n", - "Hitters.columns" - ] - }, - { - "cell_type": "markdown", - "id": "1a0a3521-be74-40df-a404-3895d80a11dc", - "metadata": {}, - "source": [ - "## Grouping variables\n", - "\n", - "A look at the [description](https://islp.readthedocs.io/en/latest/datasets/Hitters.html) of the data shows\n", - "that there are both career and 1986 offensive stats, as well as some defensive stats.\n", - "\n", - "Let's group the offensive into recent and career offensive stats, as well as a group of defensive variables." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a215e43b-7bc8-4bdd-91cf-40d717cd7978", - "metadata": {}, - "outputs": [], - "source": [ - "confounders = derived_feature(['Division', 'League', 'NewLeague'],\n", - " name='confounders')\n", - "offense_career = derived_feature(['CAtBat', 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks'],\n", - " name='offense_career')\n", - "offense_1986 = derived_feature(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks'],\n", - " name='offense_1986')\n", - "defense_1986 = derived_feature(['PutOuts', 'Assists', 'Errors'],\n", - " name='defense_1986')" - ] - }, - { - "cell_type": "markdown", - "id": "aa15fd0c-1e8a-431e-8425-c61da8439976", - "metadata": {}, - "source": [ - "We'll first do a sequential ANOVA where terms are added sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "40cd6c28", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([confounders, offense_career, defense_1986, offense_1986], intercept=True).fit(Hitters)\n", - "Y = np.array(Hitters['Salary'])\n", - "X = design.transform(Hitters)" - ] - }, - { - "cell_type": "markdown", - "id": "074120b1", - "metadata": {}, - "source": [ - "Along with a score we need to specify the search strategy. This is done through the object\n", - "`Stepwise()` in the `ISLP.models` package. The method `Stepwise.first_peak()`\n", - "runs forward stepwise until any further additions to the model do not result\n", - "in an improvement in the evaluation score. Similarly, the method `Stepwise.fixed_steps()`\n", - "runs a fixed number of steps of stepwise search." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "e65f5607", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
coefstd errtP>|t|
intercept148.218773.5952.0140.045
Division[W]-116.040440.188-2.8870.004
League[N]63.750379.0060.8070.421
NewLeague[N]-24.398978.843-0.3090.757
CAtBat-0.18870.120-1.5720.117
CHits0.16360.6650.2460.806
CHmRun-0.15171.612-0.0940.925
CRuns1.47160.7471.9710.050
CRBI0.80210.6911.1610.247
CWalks-0.81240.327-2.4810.014
PutOuts0.28270.0773.6610.000
Assists0.37550.2201.7050.089
Errors-3.29404.377-0.7530.452
AtBat-1.95090.624-3.1250.002
Hits7.43952.3633.1480.002
HmRun4.34496.1900.7020.483
Runs-2.33122.971-0.7850.433
RBI-1.06702.595-0.4110.681
Walks6.21961.8253.4090.001
\n", - "
" - ], - "text/plain": [ - " coef std err t P>|t|\n", - "intercept 148.2187 73.595 2.014 0.045\n", - "Division[W] -116.0404 40.188 -2.887 0.004\n", - "League[N] 63.7503 79.006 0.807 0.421\n", - "NewLeague[N] -24.3989 78.843 -0.309 0.757\n", - "CAtBat -0.1887 0.120 -1.572 0.117\n", - "CHits 0.1636 0.665 0.246 0.806\n", - "CHmRun -0.1517 1.612 -0.094 0.925\n", - "CRuns 1.4716 0.747 1.971 0.050\n", - "CRBI 0.8021 0.691 1.161 0.247\n", - "CWalks -0.8124 0.327 -2.481 0.014\n", - "PutOuts 0.2827 0.077 3.661 0.000\n", - "Assists 0.3755 0.220 1.705 0.089\n", - "Errors -3.2940 4.377 -0.753 0.452\n", - "AtBat -1.9509 0.624 -3.125 0.002\n", - "Hits 7.4395 2.363 3.148 0.002\n", - "HmRun 4.3449 6.190 0.702 0.483\n", - "Runs -2.3312 2.971 -0.785 0.433\n", - "RBI -1.0670 2.595 -0.411 0.681\n", - "Walks 6.2196 1.825 3.409 0.001" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "M = OLS(Y, X).fit()\n", - "summarize(M)" - ] - }, - { - "cell_type": "markdown", - "id": "29d9b55f", - "metadata": {}, - "source": [ - "We'll first produce the sequential, or Type I ANOVA results. This builds up a model sequentially and compares\n", - "two successive models." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "cfbe5b92", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
df_residssrdf_diffss_diffFPr(>F)
intercept262.05.331911e+070.0NaNNaNNaN
confounders259.05.131263e+073.02.006478e+066.7411472.144265e-04
offense_career253.03.059130e+076.02.072134e+0734.8086561.470455e-30
defense_1986250.02.730614e+073.03.285156e+0611.0371117.880207e-07
offense_1986244.02.420857e+076.03.097572e+065.2034444.648586e-05
\n", - "
" - ], - "text/plain": [ - " df_resid ssr df_diff ss_diff F \\\n", - "intercept 262.0 5.331911e+07 0.0 NaN NaN \n", - "confounders 259.0 5.131263e+07 3.0 2.006478e+06 6.741147 \n", - "offense_career 253.0 3.059130e+07 6.0 2.072134e+07 34.808656 \n", - "defense_1986 250.0 2.730614e+07 3.0 3.285156e+06 11.037111 \n", - "offense_1986 244.0 2.420857e+07 6.0 3.097572e+06 5.203444 \n", - "\n", - " Pr(>F) \n", - "intercept NaN \n", - "confounders 2.144265e-04 \n", - "offense_career 1.470455e-30 \n", - "defense_1986 7.880207e-07 \n", - "offense_1986 4.648586e-05 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = anova_lm(*[OLS(Y, D).fit() for D in design.build_sequence(Hitters, anova_type='sequential')])\n", - "df.index = design.names\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "7092f666", - "metadata": {}, - "source": [ - "We can similarly compute the Type II ANOVA results which drops each term and compares to the full model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "e2d43844", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
df_residssrdf_diffss_diffFPr(>F)
intercept244.02.420857e+071.04.024254e+054.0560764.511037e-02
confounders244.02.420857e+073.09.661738e+053.2460462.261572e-02
offense_career244.02.420857e+076.01.051025e+0717.6555965.701196e-17
defense_1986244.02.420857e+073.01.467933e+064.9318032.415732e-03
offense_1986244.02.420857e+076.03.097572e+065.2034444.648586e-05
\n", - "
" - ], - "text/plain": [ - " df_resid ssr df_diff ss_diff F \\\n", - "intercept 244.0 2.420857e+07 1.0 4.024254e+05 4.056076 \n", - "confounders 244.0 2.420857e+07 3.0 9.661738e+05 3.246046 \n", - "offense_career 244.0 2.420857e+07 6.0 1.051025e+07 17.655596 \n", - "defense_1986 244.0 2.420857e+07 3.0 1.467933e+06 4.931803 \n", - "offense_1986 244.0 2.420857e+07 6.0 3.097572e+06 5.203444 \n", - "\n", - " Pr(>F) \n", - "intercept 4.511037e-02 \n", - "confounders 2.261572e-02 \n", - "offense_career 5.701196e-17 \n", - "defense_1986 2.415732e-03 \n", - "offense_1986 4.648586e-05 " - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "D_full = design.transform(Hitters)\n", - "OLS_full = OLS(Y, D_full).fit()\n", - "dfs = []\n", - "for d in design.build_sequence(Hitters, anova_type='drop'):\n", - " dfs.append(anova_lm(OLS(Y,d).fit(), OLS_full).iloc[1:])\n", - "df = pd.concat(dfs)\n", - "df.index = design.names\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "362709ae-9558-4c4c-8f5e-f8388caf631d", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "jupytext": { - "formats": "source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyterbook/models/anova.md b/docs/jupyterbook/models/anova.md deleted file mode 100644 index 599a607..0000000 --- a/docs/jupyterbook/models/anova.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -jupytext: - formats: source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.14.5 -kernelspec: - display_name: python3 - language: python - name: python3 ---- - -# ANOVA using `ModelSpec` - - -In this lab we illustrate how to run create specific ANOVA analyses -using `ModelSpec`. - -```{code-cell} ipython3 -import numpy as np -import pandas as pd - -from statsmodels.api import OLS -from statsmodels.stats.anova import anova_lm - -from ISLP import load_data -from ISLP.models import (ModelSpec, - derived_feature, - summarize) -``` - -We will carry out two simple ANOVA analyses of the `Hitters` data. -We wish to predict a baseball player’s `Salary` on the -basis of various statistics associated with performance in the -previous year. - -```{code-cell} ipython3 -Hitters = load_data('Hitters') -np.isnan(Hitters['Salary']).sum() -``` - - - We see that `Salary` is missing for 59 players. The -`dropna()` method of data frames removes all of the rows that have missing -values in any variable (by default --- see `Hitters.dropna?`). - -```{code-cell} ipython3 -Hitters = Hitters.dropna() -Hitters.columns -``` - -## Grouping variables - -A look at the [description](https://islp.readthedocs.io/en/latest/datasets/Hitters.html) of the data shows -that there are both career and 1986 offensive stats, as well as some defensive stats. - -Let's group the offensive into recent and career offensive stats, as well as a group of defensive variables. - -```{code-cell} ipython3 -confounders = derived_feature(['Division', 'League', 'NewLeague'], - name='confounders') -offense_career = derived_feature(['CAtBat', 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks'], - name='offense_career') -offense_1986 = derived_feature(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks'], - name='offense_1986') -defense_1986 = derived_feature(['PutOuts', 'Assists', 'Errors'], - name='defense_1986') -``` - -We'll first do a sequential ANOVA where terms are added sequentially - -```{code-cell} ipython3 -design = ModelSpec([confounders, offense_career, defense_1986, offense_1986], intercept=True).fit(Hitters) -Y = np.array(Hitters['Salary']) -X = design.transform(Hitters) -``` - -Along with a score we need to specify the search strategy. This is done through the object -`Stepwise()` in the `ISLP.models` package. The method `Stepwise.first_peak()` -runs forward stepwise until any further additions to the model do not result -in an improvement in the evaluation score. Similarly, the method `Stepwise.fixed_steps()` -runs a fixed number of steps of stepwise search. - -```{code-cell} ipython3 -M = OLS(Y, X).fit() -summarize(M) -``` - -We'll first produce the sequential, or Type I ANOVA results. This builds up a model sequentially and compares -two successive models. - -```{code-cell} ipython3 -df = anova_lm(*[OLS(Y, D).fit() for D in design.build_sequence(Hitters, anova_type='sequential')]) -df.index = design.names -df -``` - -We can similarly compute the Type II ANOVA results which drops each term and compares to the full model. - -```{code-cell} ipython3 -D_full = design.transform(Hitters) -OLS_full = OLS(Y, D_full).fit() -dfs = [] -for d in design.build_sequence(Hitters, anova_type='drop'): - dfs.append(anova_lm(OLS(Y,d).fit(), OLS_full).iloc[1:]) -df = pd.concat(dfs) -df.index = design.names -df -``` - -```{code-cell} ipython3 - -``` diff --git a/docs/jupyterbook/models/spec.ipynb b/docs/jupyterbook/models/spec.ipynb deleted file mode 100644 index 6940e7d..0000000 --- a/docs/jupyterbook/models/spec.ipynb +++ /dev/null @@ -1,1926 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "typical-correlation", - "metadata": {}, - "source": [ - "# Building design matrices with `ModelSpec`\n", - "\n", - "The `ISLP` package provides a facility to build design\n", - "matrices for regression and classification tasks. It provides similar functionality to the formula\n", - "notation of `R` though uses python objects rather than specification through the special formula syntax.\n", - "\n", - "Related tools include `patsy` and `ColumnTransformer` from `sklearn.compose`. \n", - "\n", - "Perhaps the most common use is to extract some columns from a `pd.DataFrame` and \n", - "produce a design matrix, optionally with an intercept." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "sticky-desperate", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec,\n", - " summarize,\n", - " Column,\n", - " Feature,\n", - " build_columns,\n", - " ordinal)\n", - "\n", - "import statsmodels.api as sm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "devoted-antique", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "b7a2e6ab-491d-4a57-8184-a9fcccb2047b", - "metadata": {}, - "source": [ - "We'll first build a design matrix that we can use to model `Sales`\n", - "in terms of the categorical variable `ShelveLoc` and `Price`.\n", - "\n", - "We see first that `ShelveLoc` is a categorical variable:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "7d3642a6-90c6-48ad-8d35-88231b4991f8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 Bad\n", - "1 Good\n", - "2 Medium\n", - "3 Medium\n", - "4 Bad\n", - " ... \n", - "395 Good\n", - "396 Medium\n", - "397 Medium\n", - "398 Bad\n", - "399 Good\n", - "Name: ShelveLoc, Length: 400, dtype: category\n", - "Categories (3, object): ['Bad', 'Good', 'Medium']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats['ShelveLoc']" - ] - }, - { - "cell_type": "markdown", - "id": "4afa201d-4b19-4d85-9e1b-1392a54d027b", - "metadata": {}, - "source": [ - "This is recognized by `ModelSpec` and only 2 columns are added for the three levels. The\n", - "default behavior is to drop the first level of the categories. Later, \n", - "we will show other contrasts of the 3 columns can be produced. \n", - "\n", - "This simple example below illustrates how the first argument (its `terms`) is\n", - "used to construct a design matrix." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "fd5528fe-11da-4e10-8996-06085896c1a0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptShelveLoc[Good]ShelveLoc[Medium]Price
01.00.00.0120
11.01.00.083
21.00.01.080
31.00.01.097
41.00.00.0128
51.00.00.072
61.00.01.0108
71.01.00.0120
81.00.01.0124
91.00.01.0124
\n", - "
" - ], - "text/plain": [ - " intercept ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 1.0 0.0 0.0 120\n", - "1 1.0 1.0 0.0 83\n", - "2 1.0 0.0 1.0 80\n", - "3 1.0 0.0 1.0 97\n", - "4 1.0 0.0 0.0 128\n", - "5 1.0 0.0 0.0 72\n", - "6 1.0 0.0 1.0 108\n", - "7 1.0 1.0 0.0 120\n", - "8 1.0 0.0 1.0 124\n", - "9 1.0 0.0 1.0 124" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True)\n", - "X = MS.fit_transform(Carseats)\n", - "X.iloc[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "6948e1ef-3685-4840-a4f2-ef15a1bcfb69", - "metadata": {}, - "source": [ - "We note that a column of ones, named `intercept`, has been added: `intercept=True`, as\n", - "statsmodels' models need it. Its default, `True` for now, will change to `False` in ISLP 0.6\n", - "(glmstar's estimators and most scikit-learn estimators fit the intercept themselves), so pass\n", - "`intercept` explicitly: leaving it out gives a `FutureWarning`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "682d4c81-eba9-467d-a176-911a0269a21d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Good]ShelveLoc[Medium]Price
00.00.0120
11.00.083
20.01.080
30.01.097
40.00.0128
50.00.072
60.01.0108
71.00.0120
80.01.0124
90.01.0124
\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 0.0 0.0 120\n", - "1 1.0 0.0 83\n", - "2 0.0 1.0 80\n", - "3 0.0 1.0 97\n", - "4 0.0 0.0 128\n", - "5 0.0 0.0 72\n", - "6 0.0 1.0 108\n", - "7 1.0 0.0 120\n", - "8 0.0 1.0 124\n", - "9 0.0 1.0 124" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS_no1 = ModelSpec(['ShelveLoc', 'Price'], intercept=False)\n", - "MS_no1.fit_transform(Carseats)[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "54d8fd20-d8f5-44d6-9965-83e745680798", - "metadata": {}, - "source": [ - "We see that `ShelveLoc` still only contributes\n", - "two columns to the design. The `ModelSpec` object does no introspection of its arguments to effectively include an intercept term\n", - "in the column space of the design matrix.\n", - "\n", - "To include this intercept via `ShelveLoc` we can use 3 columns to encode this categorical variable. Following the nomenclature of\n", - "`R`, we call this a `Contrast` of the categorical variable." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "555734bb-2682-4721-a1cd-6fb207394b0e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Bad]ShelveLoc[Good]ShelveLoc[Medium]Price
01.00.00.0120
10.01.00.083
20.00.01.080
30.00.01.097
41.00.00.0128
51.00.00.072
60.00.01.0108
70.01.00.0120
80.00.01.0124
90.00.01.0124
\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Bad] ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 1.0 0.0 0.0 120\n", - "1 0.0 1.0 0.0 83\n", - "2 0.0 0.0 1.0 80\n", - "3 0.0 0.0 1.0 97\n", - "4 1.0 0.0 0.0 128\n", - "5 1.0 0.0 0.0 72\n", - "6 0.0 0.0 1.0 108\n", - "7 0.0 1.0 0.0 120\n", - "8 0.0 0.0 1.0 124\n", - "9 0.0 0.0 1.0 124" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from ISLP.models import contrast\n", - "shelve = contrast('ShelveLoc', None)\n", - "MS_contr = ModelSpec([shelve, 'Price'], intercept=False)\n", - "MS_contr.fit_transform(Carseats)[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "66db03cf-489c-40b6-8fac-762d66cf9932", - "metadata": {}, - "source": [ - "This example above illustrates that columns need not be identified by name in `terms`. The basic\n", - "role of an item in the `terms` sequence is a description of how to extract a column\n", - "from a columnar data object, usually a `pd.DataFrame`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "852ee40e-05d2-4785-ab7d-968fb087f3c0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Column(idx='ShelveLoc', name='ShelveLoc', is_categorical=True, is_ordinal=False, columns=(), encoder=Contrast(method=None))" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "shelve" - ] - }, - { - "cell_type": "markdown", - "id": "b3be8808-1dbf-4154-882b-f61656a2ed4e", - "metadata": {}, - "source": [ - "The `Column` object can be used to directly extract relevant columns from a `pd.DataFrame`. If the `encoder` field is not\n", - "`None`, then the extracted columns will be passed through `encoder`.\n", - "The `get_columns` method produces these columns as well as names for the columns.\n", - "The encoder of `shelve` itself has not been fit: fitting `MS_contr` fit a copy of it,\n", - "so that `shelve` can be reused in other specifications. We ask `get_columns` to fit it with `fit=True`." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "0ebadfc0-0ea2-4abc-aac6-ef78be227ce1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[1., 0., 0.],\n", - " [0., 1., 0.],\n", - " [0., 0., 1.],\n", - " ...,\n", - " [0., 0., 1.],\n", - " [1., 0., 0.],\n", - " [0., 1., 0.]], shape=(400, 3)),\n", - " ['ShelveLoc[Bad]', 'ShelveLoc[Good]', 'ShelveLoc[Medium]'])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "shelve.get_columns(Carseats, fit=True)" - ] - }, - { - "cell_type": "markdown", - "id": "269e6d18-4ae4-4a77-8498-90281ae7c803", - "metadata": {}, - "source": [ - "Let's now fit a simple OLS model with this design." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "411238d0-dd36-4878-a869-e8ce0ada099c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
coefstd errtP>|t|
ShelveLoc[Bad]12.00180.50323.8390.0
ShelveLoc[Good]16.89760.52232.3860.0
ShelveLoc[Medium]13.86380.48728.4670.0
Price-0.05670.004-13.9670.0
\n", - "
" - ], - "text/plain": [ - " coef std err t P>|t|\n", - "ShelveLoc[Bad] 12.0018 0.503 23.839 0.0\n", - "ShelveLoc[Good] 16.8976 0.522 32.386 0.0\n", - "ShelveLoc[Medium] 13.8638 0.487 28.467 0.0\n", - "Price -0.0567 0.004 -13.967 0.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X = MS_contr.transform(Carseats)\n", - "Y = Carseats['Sales']\n", - "M_ols = sm.OLS(Y, X).fit()\n", - "summarize(M_ols)" - ] - }, - { - "cell_type": "markdown", - "id": "40ddf68e-7d58-4e30-93a8-5b7fe840d37a", - "metadata": {}, - "source": [ - "## Interactions\n", - "\n", - "One of the common uses of formulae in `R` is to specify interactions between variables.\n", - "This is done in `ModelSpec` by including a tuple in the `terms` argument." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "3f5e314c-7a7f-4e8d-bb07-295beb42c728", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptShelveLoc[Bad]:PriceShelveLoc[Good]:PriceShelveLoc[Medium]:PricePrice
01.0120.00.00.0120
11.00.083.00.083
21.00.00.080.080
31.00.00.097.097
41.0128.00.00.0128
51.072.00.00.072
61.00.00.0108.0108
71.00.0120.00.0120
81.00.00.0124.0124
91.00.00.0124.0124
\n", - "
" - ], - "text/plain": [ - " intercept ShelveLoc[Bad]:Price ShelveLoc[Good]:Price \\\n", - "0 1.0 120.0 0.0 \n", - "1 1.0 0.0 83.0 \n", - "2 1.0 0.0 0.0 \n", - "3 1.0 0.0 0.0 \n", - "4 1.0 128.0 0.0 \n", - "5 1.0 72.0 0.0 \n", - "6 1.0 0.0 0.0 \n", - "7 1.0 0.0 120.0 \n", - "8 1.0 0.0 0.0 \n", - "9 1.0 0.0 0.0 \n", - "\n", - " ShelveLoc[Medium]:Price Price \n", - "0 0.0 120 \n", - "1 0.0 83 \n", - "2 80.0 80 \n", - "3 97.0 97 \n", - "4 0.0 128 \n", - "5 0.0 72 \n", - "6 108.0 108 \n", - "7 0.0 120 \n", - "8 124.0 124 \n", - "9 124.0 124 " - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ModelSpec([(shelve, 'Price'), 'Price'], intercept=True).fit_transform(Carseats).iloc[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "3f85fcb2-f0ef-4c1b-a89f-fcf083937274", - "metadata": {}, - "source": [ - "The above design matrix is clearly rank deficient, as `ModelSpec` has not inspected the formula\n", - "and attempted to produce a corresponding matrix that may or may not match a user's intent." - ] - }, - { - "cell_type": "markdown", - "id": "excellent-hamilton", - "metadata": {}, - "source": [ - "## Ordinal variables\n", - "\n", - "Ordered categoricals, such as the income bands produced by `pd.cut()` below,\n", - "are encoded with contrasts like any other categorical, in the order of their\n", - "categories." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "going-administrator", - "metadata": {}, - "outputs": [], - "source": [ - "Carseats['OIncome'] = pd.cut(Carseats['Income'], \n", - " [0,50,90,200], \n", - " labels=['L','M','H'])\n", - "MS_order = ModelSpec(['OIncome'], intercept=True).fit(Carseats)" - ] - }, - { - "cell_type": "markdown", - "id": "5e1defb1-071b-4751-9358-b8d2f0b3412e", - "metadata": {}, - "source": [ - "Part of the `fit` method of `ModelSpec` involves inspection of the columns of `Carseats`. \n", - "The results of that inspection can be found in the `column_info_` attribute:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "050fb4ae-648d-429d-9cb2-8423ad9707d7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Sales': Column(idx='Sales', name='Sales', is_categorical=False, is_ordinal=False, columns=('Sales',), encoder=None),\n", - " 'CompPrice': Column(idx='CompPrice', name='CompPrice', is_categorical=False, is_ordinal=False, columns=('CompPrice',), encoder=None),\n", - " 'Income': Column(idx='Income', name='Income', is_categorical=False, is_ordinal=False, columns=('Income',), encoder=None),\n", - " 'Advertising': Column(idx='Advertising', name='Advertising', is_categorical=False, is_ordinal=False, columns=('Advertising',), encoder=None),\n", - " 'Population': Column(idx='Population', name='Population', is_categorical=False, is_ordinal=False, columns=('Population',), encoder=None),\n", - " 'Price': Column(idx='Price', name='Price', is_categorical=False, is_ordinal=False, columns=('Price',), encoder=None),\n", - " 'ShelveLoc': Column(idx='ShelveLoc', name='ShelveLoc', is_categorical=True, is_ordinal=False, columns=('ShelveLoc[Good]', 'ShelveLoc[Medium]'), encoder=Contrast()),\n", - " 'Age': Column(idx='Age', name='Age', is_categorical=False, is_ordinal=False, columns=('Age',), encoder=None),\n", - " 'Education': Column(idx='Education', name='Education', is_categorical=False, is_ordinal=False, columns=('Education',), encoder=None),\n", - " 'Urban': Column(idx='Urban', name='Urban', is_categorical=True, is_ordinal=False, columns=('Urban[Yes]',), encoder=Contrast()),\n", - " 'US': Column(idx='US', name='US', is_categorical=True, is_ordinal=False, columns=('US[Yes]',), encoder=Contrast()),\n", - " 'OIncome': Column(idx='OIncome', name='OIncome', is_categorical=True, is_ordinal=True, columns=('OIncome[M]', 'OIncome[H]'), encoder=Contrast())}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS_order.column_info_" - ] - }, - { - "cell_type": "markdown", - "id": "7d2a582d", - "metadata": {}, - "source": [ - "To instead encode an ordinal variable by a single column holding the\n", - "position of its level, use `ordinal()`. The order comes from an ordered\n", - "categorical, as here, or can be given with `levels=[...]`; for other\n", - "columns, such as strings, the order would be ambiguous and `ordinal()`\n", - "raises an error." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "35913f64", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptordinal(OIncome)
01.01.0
11.00.0
21.00.0
31.02.0
41.01.0
\n", - "
" - ], - "text/plain": [ - " intercept ordinal(OIncome)\n", - "0 1.0 1.0\n", - "1 1.0 0.0\n", - "2 1.0 0.0\n", - "3 1.0 2.0\n", - "4 1.0 1.0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ModelSpec([ordinal('OIncome')], intercept=True).fit_transform(Carseats).iloc[:5]" - ] - }, - { - "cell_type": "markdown", - "id": "debf7e2e-0a9d-451b-866c-66c0df9f43e5", - "metadata": {}, - "source": [ - "## Structure of a `ModelSpec`\n", - "\n", - "The first argument to `ModelSpec` is stored as the `terms` attribute. Under the hood,\n", - "this sequence is inspected to produce the `terms_` attribute which specify the objects\n", - "that will ultimately create the design matrix." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "ea51e988-0857-4d49-9987-d7531b34a233", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Feature(variables=('ShelveLoc',), name='ShelveLoc', encoder=None, use_transform=True, pure_columns=True, override_encoder_colnames=False),\n", - " Feature(variables=('Price',), name='Price', encoder=None, use_transform=True, pure_columns=True, override_encoder_colnames=False)]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True)\n", - "MS.fit(Carseats)\n", - "MS.terms_" - ] - }, - { - "cell_type": "markdown", - "id": "warming-mobile", - "metadata": {}, - "source": [ - "Each element of `terms_` should be a `Feature` which describes a set of columns to be extracted from\n", - "a columnar data form as well as possible a possible encoder." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "59214a70-1e6b-41c4-9f44-a92d340723c9", - "metadata": {}, - "outputs": [], - "source": [ - "shelve_var = MS.terms_[0]" - ] - }, - { - "cell_type": "markdown", - "id": "5fed3ea2-ff50-4e5d-819d-a948f121f9d3", - "metadata": {}, - "source": [ - "We can find the columns associated to each term using the `build_columns` method of `ModelSpec`:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "5e25ef64-497d-4f42-9f20-3d4a320cda23", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Good]ShelveLoc[Medium]
00.00.0
11.00.0
20.01.0
30.01.0
40.00.0
.........
3951.00.0
3960.01.0
3970.01.0
3980.00.0
3991.00.0
\n", - "

400 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Good] ShelveLoc[Medium]\n", - "0 0.0 0.0\n", - "1 1.0 0.0\n", - "2 0.0 1.0\n", - "3 0.0 1.0\n", - "4 0.0 0.0\n", - ".. ... ...\n", - "395 1.0 0.0\n", - "396 0.0 1.0\n", - "397 0.0 1.0\n", - "398 0.0 0.0\n", - "399 1.0 0.0\n", - "\n", - "[400 rows x 2 columns]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df, names = build_columns(MS.column_info_,\n", - " Carseats, \n", - " shelve_var)\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "63edf7a2-e776-45b0-b434-d676d7e13dbd", - "metadata": {}, - "source": [ - "The design matrix is constructed by running through `terms_` and concatenating the corresponding columns." - ] - }, - { - "cell_type": "markdown", - "id": "former-spring", - "metadata": {}, - "source": [ - "### `Feature` objects\n", - "\n", - "Note that `Feature` objects have a tuple of `variables` as well as an `encoder` attribute. The\n", - "tuple of `variables` first creates a concatenated dataframe from all corresponding variables and then\n", - "is run through `encoder.transform`. The `encoder.fit` method of each `Feature` is run once during \n", - "the call to `ModelSpec.fit`." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "floral-liabilities", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
PriceIncomeOIncome
0120.073.02.0
183.048.01.0
280.035.01.0
397.0100.00.0
4128.064.02.0
............
395128.0108.00.0
396120.023.01.0
397159.026.01.0
39895.079.02.0
399120.037.01.0
\n", - "

400 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " Price Income OIncome\n", - "0 120.0 73.0 2.0\n", - "1 83.0 48.0 1.0\n", - "2 80.0 35.0 1.0\n", - "3 97.0 100.0 0.0\n", - "4 128.0 64.0 2.0\n", - ".. ... ... ...\n", - "395 128.0 108.0 0.0\n", - "396 120.0 23.0 1.0\n", - "397 159.0 26.0 1.0\n", - "398 95.0 79.0 2.0\n", - "399 120.0 37.0 1.0\n", - "\n", - "[400 rows x 3 columns]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=None)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " new_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "reasonable-canadian", - "metadata": {}, - "source": [ - "Let's now transform these columns with an encoder. Within `ModelSpec` we will first build the\n", - "arrays above and then call `pca.fit` and finally `pca.transform` within `design.build_columns`." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "imported-measure", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
mynewvar[0]mynewvar[1]
0-3.595740-4.850530
115.07040135.706773
227.41222840.772377
3-33.98304813.468087
46.580644-11.287452
.........
395-36.856308-18.418138
39645.7315203.243768
39749.087659-35.727136
398-13.56517818.847760
39931.9170720.976615
\n", - "

400 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " mynewvar[0] mynewvar[1]\n", - "0 -3.595740 -4.850530\n", - "1 15.070401 35.706773\n", - "2 27.412228 40.772377\n", - "3 -33.983048 13.468087\n", - "4 6.580644 -11.287452\n", - ".. ... ...\n", - "395 -36.856308 -18.418138\n", - "396 45.731520 3.243768\n", - "397 49.087659 -35.727136\n", - "398 -13.565178 18.847760\n", - "399 31.917072 0.976615\n", - "\n", - "[400 rows x 2 columns]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.decomposition import PCA\n", - "pca = PCA(n_components=2)\n", - "pca.fit(build_columns(MS.column_info_, Carseats, new_var)[0]) # this is done within `ModelSpec.fit`\n", - "pca_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=pca)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " pca_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "institutional-burden", - "metadata": {}, - "source": [ - "The elements of the `variables` attribute may be column identifiers ( `\"Price\"`), `Column` instances (`price`)\n", - "or `Feature` instances (`pca_var`)." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "western-bloom", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IncomePricemynewvar[0]mynewvar[1]
073.0120.0-3.595740-4.850530
148.083.015.07040135.706773
235.080.027.41222840.772377
3100.097.0-33.98304813.468087
464.0128.06.580644-11.287452
...............
395108.0128.0-36.856308-18.418138
39623.0120.045.7315203.243768
39726.0159.049.087659-35.727136
39879.095.0-13.56517818.847760
39937.0120.031.9170720.976615
\n", - "

400 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " Income Price mynewvar[0] mynewvar[1]\n", - "0 73.0 120.0 -3.595740 -4.850530\n", - "1 48.0 83.0 15.070401 35.706773\n", - "2 35.0 80.0 27.412228 40.772377\n", - "3 100.0 97.0 -33.983048 13.468087\n", - "4 64.0 128.0 6.580644 -11.287452\n", - ".. ... ... ... ...\n", - "395 108.0 128.0 -36.856308 -18.418138\n", - "396 23.0 120.0 45.731520 3.243768\n", - "397 26.0 159.0 49.087659 -35.727136\n", - "398 79.0 95.0 -13.565178 18.847760\n", - "399 37.0 120.0 31.917072 0.976615\n", - "\n", - "[400 rows x 4 columns]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "price = MS.column_info_['Price']\n", - "fancy_var = Feature(('Income', price, pca_var), name='fancy', encoder=None)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " fancy_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "e289feba-e3f5-48e0-9e29-cdd88d7f9923", - "metadata": {}, - "source": [ - "## Predicting at new points" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "6efed2fa-9e5d-429c-a8d9-ac544cab2b41", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "intercept 12.661546\n", - "Price -0.052213\n", - "Income 0.012829\n", - "dtype: float64" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['Price', 'Income'], intercept=True).fit(Carseats)\n", - "X = MS.transform(Carseats)\n", - "Y = Carseats['Sales']\n", - "M_ols = sm.OLS(Y, X).fit()\n", - "M_ols.params" - ] - }, - { - "cell_type": "markdown", - "id": "e6b4609b-fcb2-4cc2-b630-509df4c87546", - "metadata": {}, - "source": [ - "As `ModelSpec` is a transformer, it can be evaluated at new feature values.\n", - "Constructing the design matrix at any values is carried out by the `transform` method." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "8784b0e8-ce53-4a90-aee6-b935834295c7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([10.70130676, 10.307465 ])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_data = pd.DataFrame({'Price':[40, 50], 'Income':[10, 20]})\n", - "new_X = MS.transform(new_data)\n", - "M_ols.get_prediction(new_X).predicted_mean" - ] - }, - { - "cell_type": "markdown", - "id": "signal-yahoo", - "metadata": {}, - "source": [ - "## Using `np.ndarray`\n", - "\n", - "As the basic model is to concatenate columns extracted from a columnar data\n", - "representation, one *can* use `np.ndarray` as the column data. In this case,\n", - "columns will be selected by integer indices. \n", - "\n", - "### Caveats using `np.ndarray`\n", - "\n", - "If the `terms` only refer to a few columns of the data frame, the `transform` method only needs a dataframe with those columns.\n", - "However,\n", - "unless all features are floats, `np.ndarray` will default to a dtype of `object`, complicating issues.\n", - "\n", - "However, if we had used an `np.ndarray`, the column identifiers would be integers identifying specific columns so,\n", - "in order to work correctly, `transform` would need another `np.ndarray` where the columns have the same meaning. \n", - "\n", - "We illustrate this below, where we build a model from `Price` and `Income` for `Sales` and want to find predictions at new\n", - "values of `Price` and `Location`. We first find the predicitions using `pd.DataFrame` and then illustrate the difficulties\n", - "in using `np.ndarray`." - ] - }, - { - "cell_type": "markdown", - "id": "e7ffdd07-4d6b-4a4c-ab38-ad1270e85de6", - "metadata": {}, - "source": [ - "We will refit this model, using `ModelSpec` with an `np.ndarray` instead" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "4fec9030-7445-48be-a15f-2ac5a789e717", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 1., 120., 73.],\n", - " [ 1., 83., 48.],\n", - " [ 1., 80., 35.],\n", - " ...,\n", - " [ 1., 159., 26.],\n", - " [ 1., 95., 79.],\n", - " [ 1., 120., 37.]])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats_np = np.asarray(Carseats[['Price', 'Education', 'Income']])\n", - "MS_np = ModelSpec([0,2], intercept=True).fit(Carseats_np)\n", - "MS_np.transform(Carseats_np)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "c864e365-2476-4ca6-9d27-625cac2b2271", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "const 12.661546\n", - "x1 -0.052213\n", - "x2 0.012829\n", - "dtype: float64" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "M_ols_np = sm.OLS(Y, MS_np.transform(Carseats_np)).fit()\n", - "M_ols_np.params" - ] - }, - { - "cell_type": "markdown", - "id": "undefined-sacrifice", - "metadata": {}, - "source": [ - "Now, let's consider finding the design matrix at new points. \n", - "When using `pd.DataFrame` we only need to supply the `transform` method\n", - "a data frame with columns implicated in the `terms` argument (in this case, `Price` and `Income`). \n", - "\n", - "However, when using `np.ndarray` with integers as indices, `Price` was column 0 and `Income` was column 2. The only\n", - "sensible way to produce a return for predict is to extract its 0th and 2nd columns. Note this means\n", - "that the meaning of columns in an `np.ndarray` provided to `transform` essentially must be identical to those\n", - "passed to `fit`. To guard against this, `transform` requires an `np.ndarray` with as many columns as it was fit with." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "incredible-concert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "X has 2 features, but ModelSpec is expecting 3 features as input.\n" - ] - } - ], - "source": [ - "try:\n", - " new_D = np.array([[40,50], [10,20]]).T\n", - " new_X = MS_np.transform(new_D)\n", - "except ValueError as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "id": "allied-botswana", - "metadata": {}, - "source": [ - "Ultimately, `M` expects 3 columns for new predictions because it was fit\n", - "with a matrix having 3 columns (the first representing an intercept).\n", - "\n", - "We might be tempted to try as with the `pd.DataFrame` and produce\n", - "an `np.ndarray` with only the necessary variables." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "stunning-container", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 1. 40. 10.]\n", - " [ 1. 50. 20.]]\n" - ] - }, - { - "data": { - "text/plain": [ - "array([10.70130676, 10.307465 ])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_D = np.array([[40,50], [np.nan, np.nan], [10,20]]).T\n", - "new_X = MS_np.transform(new_D)\n", - "print(new_X)\n", - "M_ols.get_prediction(new_X).predicted_mean" - ] - }, - { - "cell_type": "markdown", - "id": "specific-tobacco", - "metadata": {}, - "source": [ - "For more complicated design contructions ensuring the columns of `new_D` match that of the original data will be more cumbersome. We expect\n", - "then that `pd.DataFrame` (or a columnar data representation with similar API) will likely be easier to use with `ModelSpec`." - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyterbook/models/spec.md b/docs/jupyterbook/models/spec.md deleted file mode 100644 index aff72fe..0000000 --- a/docs/jupyterbook/models/spec.md +++ /dev/null @@ -1,317 +0,0 @@ ---- -jupytext: - formats: source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.14.5 -kernelspec: - display_name: Python 3 (ipykernel) - language: python - name: python3 ---- - -# Building design matrices with `ModelSpec` - -The `ISLP` package provides a facility to build design -matrices for regression and classification tasks. It provides similar functionality to the formula -notation of `R` though uses python objects rather than specification through the special formula syntax. - -Related tools include `patsy` and `ColumnTransformer` from `sklearn.compose`. - -Perhaps the most common use is to extract some columns from a `pd.DataFrame` and -produce a design matrix, optionally with an intercept. - -```{code-cell} ipython3 -import pandas as pd -import numpy as np - -from ISLP import load_data -from ISLP.models import (ModelSpec, - summarize, - Column, - Feature, - build_columns, - ordinal) - -import statsmodels.api as sm -``` - -```{code-cell} ipython3 -Carseats = load_data('Carseats') -Carseats.columns -``` - -We'll first build a design matrix that we can use to model `Sales` -in terms of the categorical variable `ShelveLoc` and `Price`. - -We see first that `ShelveLoc` is a categorical variable: - -```{code-cell} ipython3 -Carseats['ShelveLoc'] -``` - -This is recognized by `ModelSpec` and only 2 columns are added for the three levels. The -default behavior is to drop the first level of the categories. Later, -we will show other contrasts of the 3 columns can be produced. - -This simple example below illustrates how the first argument (its `terms`) is -used to construct a design matrix. - -```{code-cell} ipython3 -MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True) -X = MS.fit_transform(Carseats) -X.iloc[:10] -``` - -We note that a column of ones, named `intercept`, has been added: `intercept=True`, as -statsmodels' models need it. Its default, `True` for now, will change to `False` in ISLP 0.6 -(glmstar's estimators and most scikit-learn estimators fit the intercept themselves), so pass -`intercept` explicitly: leaving it out gives a `FutureWarning`. - -```{code-cell} ipython3 -MS_no1 = ModelSpec(['ShelveLoc', 'Price'], intercept=False) -MS_no1.fit_transform(Carseats)[:10] -``` - -We see that `ShelveLoc` still only contributes -two columns to the design. The `ModelSpec` object does no introspection of its arguments to effectively include an intercept term -in the column space of the design matrix. - -To include this intercept via `ShelveLoc` we can use 3 columns to encode this categorical variable. Following the nomenclature of -`R`, we call this a `Contrast` of the categorical variable. - -```{code-cell} ipython3 -from ISLP.models import contrast -shelve = contrast('ShelveLoc', None) -MS_contr = ModelSpec([shelve, 'Price'], intercept=False) -MS_contr.fit_transform(Carseats)[:10] -``` - -This example above illustrates that columns need not be identified by name in `terms`. The basic -role of an item in the `terms` sequence is a description of how to extract a column -from a columnar data object, usually a `pd.DataFrame`. - -```{code-cell} ipython3 -shelve -``` - -The `Column` object can be used to directly extract relevant columns from a `pd.DataFrame`. If the `encoder` field is not -`None`, then the extracted columns will be passed through `encoder`. -The `get_columns` method produces these columns as well as names for the columns. -The encoder of `shelve` itself has not been fit: fitting `MS_contr` fit a copy of it, -so that `shelve` can be reused in other specifications. We ask `get_columns` to fit it with `fit=True`. - -```{code-cell} ipython3 -shelve.get_columns(Carseats, fit=True) -``` - -Let's now fit a simple OLS model with this design. - -```{code-cell} ipython3 -X = MS_contr.transform(Carseats) -Y = Carseats['Sales'] -M_ols = sm.OLS(Y, X).fit() -summarize(M_ols) -``` - -## Interactions - -One of the common uses of formulae in `R` is to specify interactions between variables. -This is done in `ModelSpec` by including a tuple in the `terms` argument. - -```{code-cell} ipython3 -ModelSpec([(shelve, 'Price'), 'Price'], intercept=True).fit_transform(Carseats).iloc[:10] -``` - -The above design matrix is clearly rank deficient, as `ModelSpec` has not inspected the formula -and attempted to produce a corresponding matrix that may or may not match a user's intent. - -+++ - -## Ordinal variables - -Ordered categoricals, such as the income bands produced by `pd.cut()` below, -are encoded with contrasts like any other categorical, in the order of their -categories. - -```{code-cell} ipython3 -Carseats['OIncome'] = pd.cut(Carseats['Income'], - [0,50,90,200], - labels=['L','M','H']) -MS_order = ModelSpec(['OIncome'], intercept=True).fit(Carseats) -``` - -Part of the `fit` method of `ModelSpec` involves inspection of the columns of `Carseats`. -The results of that inspection can be found in the `column_info_` attribute: - -```{code-cell} ipython3 -MS_order.column_info_ -``` - -To instead encode an ordinal variable by a single column holding the -position of its level, use `ordinal()`. The order comes from an ordered -categorical, as here, or can be given with `levels=[...]`; for other -columns, such as strings, the order would be ambiguous and `ordinal()` -raises an error. - -```{code-cell} ipython3 -ModelSpec([ordinal('OIncome')], intercept=True).fit_transform(Carseats).iloc[:5] -``` - -## Structure of a `ModelSpec` - -The first argument to `ModelSpec` is stored as the `terms` attribute. Under the hood, -this sequence is inspected to produce the `terms_` attribute which specify the objects -that will ultimately create the design matrix. - -```{code-cell} ipython3 -MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True) -MS.fit(Carseats) -MS.terms_ -``` - -Each element of `terms_` should be a `Feature` which describes a set of columns to be extracted from -a columnar data form as well as possible a possible encoder. - -```{code-cell} ipython3 -shelve_var = MS.terms_[0] -``` - -We can find the columns associated to each term using the `build_columns` method of `ModelSpec`: - -```{code-cell} ipython3 -df, names = build_columns(MS.column_info_, - Carseats, - shelve_var) -df -``` - -The design matrix is constructed by running through `terms_` and concatenating the corresponding columns. - -+++ - -### `Feature` objects - -Note that `Feature` objects have a tuple of `variables` as well as an `encoder` attribute. The -tuple of `variables` first creates a concatenated dataframe from all corresponding variables and then -is run through `encoder.transform`. The `encoder.fit` method of each `Feature` is run once during -the call to `ModelSpec.fit`. - -```{code-cell} ipython3 -new_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=None) -build_columns(MS.column_info_, - Carseats, - new_var)[0] -``` - -Let's now transform these columns with an encoder. Within `ModelSpec` we will first build the -arrays above and then call `pca.fit` and finally `pca.transform` within `design.build_columns`. - -```{code-cell} ipython3 -from sklearn.decomposition import PCA -pca = PCA(n_components=2) -pca.fit(build_columns(MS.column_info_, Carseats, new_var)[0]) # this is done within `ModelSpec.fit` -pca_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=pca) -build_columns(MS.column_info_, - Carseats, - pca_var)[0] -``` - -The elements of the `variables` attribute may be column identifiers ( `"Price"`), `Column` instances (`price`) -or `Feature` instances (`pca_var`). - -```{code-cell} ipython3 -price = MS.column_info_['Price'] -fancy_var = Feature(('Income', price, pca_var), name='fancy', encoder=None) -build_columns(MS.column_info_, - Carseats, - fancy_var)[0] -``` - -## Predicting at new points - -```{code-cell} ipython3 -MS = ModelSpec(['Price', 'Income'], intercept=True).fit(Carseats) -X = MS.transform(Carseats) -Y = Carseats['Sales'] -M_ols = sm.OLS(Y, X).fit() -M_ols.params -``` - -As `ModelSpec` is a transformer, it can be evaluated at new feature values. -Constructing the design matrix at any values is carried out by the `transform` method. - -```{code-cell} ipython3 -new_data = pd.DataFrame({'Price':[40, 50], 'Income':[10, 20]}) -new_X = MS.transform(new_data) -M_ols.get_prediction(new_X).predicted_mean -``` - -## Using `np.ndarray` - -As the basic model is to concatenate columns extracted from a columnar data -representation, one *can* use `np.ndarray` as the column data. In this case, -columns will be selected by integer indices. - -### Caveats using `np.ndarray` - -If the `terms` only refer to a few columns of the data frame, the `transform` method only needs a dataframe with those columns. -However, -unless all features are floats, `np.ndarray` will default to a dtype of `object`, complicating issues. - -However, if we had used an `np.ndarray`, the column identifiers would be integers identifying specific columns so, -in order to work correctly, `transform` would need another `np.ndarray` where the columns have the same meaning. - -We illustrate this below, where we build a model from `Price` and `Income` for `Sales` and want to find predictions at new -values of `Price` and `Location`. We first find the predicitions using `pd.DataFrame` and then illustrate the difficulties -in using `np.ndarray`. - -+++ - -We will refit this model, using `ModelSpec` with an `np.ndarray` instead - -```{code-cell} ipython3 -Carseats_np = np.asarray(Carseats[['Price', 'Education', 'Income']]) -MS_np = ModelSpec([0,2], intercept=True).fit(Carseats_np) -MS_np.transform(Carseats_np) -``` - -```{code-cell} ipython3 -M_ols_np = sm.OLS(Y, MS_np.transform(Carseats_np)).fit() -M_ols_np.params -``` - -Now, let's consider finding the design matrix at new points. -When using `pd.DataFrame` we only need to supply the `transform` method -a data frame with columns implicated in the `terms` argument (in this case, `Price` and `Income`). - -However, when using `np.ndarray` with integers as indices, `Price` was column 0 and `Income` was column 2. The only -sensible way to produce a return for predict is to extract its 0th and 2nd columns. Note this means -that the meaning of columns in an `np.ndarray` provided to `transform` essentially must be identical to those -passed to `fit`. To guard against this, `transform` requires an `np.ndarray` with as many columns as it was fit with. - -```{code-cell} ipython3 -try: - new_D = np.array([[40,50], [10,20]]).T - new_X = MS_np.transform(new_D) -except ValueError as e: - print(e) -``` - -Ultimately, `M` expects 3 columns for new predictions because it was fit -with a matrix having 3 columns (the first representing an intercept). - -We might be tempted to try as with the `pd.DataFrame` and produce -an `np.ndarray` with only the necessary variables. - -```{code-cell} ipython3 -new_D = np.array([[40,50], [np.nan, np.nan], [10,20]]).T -new_X = MS_np.transform(new_D) -print(new_X) -M_ols.get_prediction(new_X).predicted_mean -``` - -For more complicated design contructions ensuring the columns of `new_D` match that of the original data will be more cumbersome. We expect -then that `pd.DataFrame` (or a columnar data representation with similar API) will likely be easier to use with `ModelSpec`. diff --git a/docs/jupyterbook/transforms/PCA.ipynb b/docs/jupyterbook/transforms/PCA.ipynb deleted file mode 100644 index a71eafe..0000000 --- a/docs/jupyterbook/transforms/PCA.ipynb +++ /dev/null @@ -1,244 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "25478d81", - "metadata": {}, - "source": [ - "# Derived features: using PCA on a subset of columns\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of transformers applied to features." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "6d8987d9", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec, \n", - " pca, \n", - " Feature, \n", - " derived_feature,\n", - " build_columns)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "6674de9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "da655f52-6f3a-4762-95ed-9b02e158bf16", - "metadata": {}, - "source": [ - "Let's create a `ModelSpec` that is aware of all of the relevant columns." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "c9afb776-ae2b-48e0-9df8-e35dc7a81327", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec(Carseats.columns.drop(['Sales']), intercept=True).fit(Carseats)" - ] - }, - { - "cell_type": "markdown", - "id": "fff603bf", - "metadata": {}, - "source": [ - "Suppose we want to make a `Feature` representing the first 3 principal components of the\n", - " features `['CompPrice', 'Income', 'Advertising', 'Population', 'Price']`." - ] - }, - { - "cell_type": "markdown", - "id": "eab49ad1-3957-478f-8a76-28a8f58551e9", - "metadata": {}, - "source": [ - "We first make a `Feature` that represents these five features columns, then `pca`\n", - "can be used to compute a new `Feature` that returns the first three principal components." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "3e6901e9-470d-41a3-8a7d-64a8da7d80fa", - "metadata": {}, - "outputs": [], - "source": [ - "grouped = Feature(('CompPrice', 'Income', 'Advertising', 'Population', 'Price'), name='grouped', encoder=None)\n", - "sklearn_pca = PCA(n_components=3, whiten=True)" - ] - }, - { - "cell_type": "markdown", - "id": "b45655a3-393d-4b4c-b754-cda61ed0e014", - "metadata": {}, - "source": [ - "We can now fit `sklearn_pca` and create our new feature." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "6cfe8861-ad07-47b9-95d1-5d5513ff6fbe", - "metadata": {}, - "outputs": [], - "source": [ - "grouped_features = build_columns(design.column_info_,\n", - " Carseats,\n", - " grouped)[0]\n", - "sklearn_pca.fit(grouped_features) \n", - "pca_var = derived_feature(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'],\n", - " name='pca(grouped)', encoder=sklearn_pca)\n", - "derived_features, _ = build_columns(design.column_info_,\n", - " Carseats, \n", - " pca_var,\n", - " encoders=design.encoders_)" - ] - }, - { - "cell_type": "markdown", - "id": "1a882cbc-8e90-4779-b14f-6fd49f5651df", - "metadata": {}, - "source": [ - "## Helper function\n", - "\n", - "The function `pca` encompasses these steps into a single function for convenience." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9f4b0955", - "metadata": {}, - "outputs": [], - "source": [ - "group_pca = pca(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'], \n", - " n_components=3, \n", - " whiten=True, \n", - " name='grouped')" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "6b382699-eb86-457f-8e91-09a63eb21d49", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['pca(grouped, n_components=3, whiten=True)[0]',\n", - " 'pca(grouped, n_components=3, whiten=True)[1]',\n", - " 'pca(grouped, n_components=3, whiten=True)[2]'],\n", - " dtype='object')" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pca_design = ModelSpec([group_pca], intercept=False)\n", - "ISLP_features = pca_design.fit_transform(Carseats)\n", - "ISLP_features.columns" - ] - }, - { - "cell_type": "markdown", - "id": "598e278c", - "metadata": {}, - "source": [ - "## Direct comparison" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "4a8d9b28", - "metadata": {}, - "outputs": [], - "source": [ - "X = np.asarray(Carseats[['CompPrice', 'Income', 'Advertising', 'Population', 'Price']])\n", - "sklearn_features = sklearn_pca.fit_transform(X)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "6efa6c67-86e1-4f51-86c2-25c838a90bf4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(4.073428490498941e-14, 0.0)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(ISLP_features - sklearn_features), np.linalg.norm(ISLP_features - np.asarray(derived_features))" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyterbook/transforms/PCA.md b/docs/jupyterbook/transforms/PCA.md deleted file mode 100644 index a56d340..0000000 --- a/docs/jupyterbook/transforms/PCA.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -jupytext: - formats: source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.14.5 -kernelspec: - display_name: python3 - language: python - name: python3 ---- - -# Derived features: using PCA on a subset of columns - -The modelling tools included in `ISLP` allow for -construction of transformers applied to features. - -```{code-cell} ipython3 -import numpy as np -from sklearn.decomposition import PCA - -from ISLP import load_data -from ISLP.models import (ModelSpec, - pca, - Feature, - derived_feature, - build_columns) -``` - -```{code-cell} ipython3 -Carseats = load_data('Carseats') -Carseats.columns -``` - -Let's create a `ModelSpec` that is aware of all of the relevant columns. - -```{code-cell} ipython3 -design = ModelSpec(Carseats.columns.drop(['Sales']), intercept=True).fit(Carseats) -``` - -Suppose we want to make a `Feature` representing the first 3 principal components of the - features `['CompPrice', 'Income', 'Advertising', 'Population', 'Price']`. - -+++ - -We first make a `Feature` that represents these five features columns, then `pca` -can be used to compute a new `Feature` that returns the first three principal components. - -```{code-cell} ipython3 -grouped = Feature(('CompPrice', 'Income', 'Advertising', 'Population', 'Price'), name='grouped', encoder=None) -sklearn_pca = PCA(n_components=3, whiten=True) -``` - -We can now fit `sklearn_pca` and create our new feature. - -```{code-cell} ipython3 -grouped_features = build_columns(design.column_info_, - Carseats, - grouped)[0] -sklearn_pca.fit(grouped_features) -pca_var = derived_feature(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'], - name='pca(grouped)', encoder=sklearn_pca) -derived_features, _ = build_columns(design.column_info_, - Carseats, - pca_var, - encoders=design.encoders_) -``` - -## Helper function - -The function `pca` encompasses these steps into a single function for convenience. - -```{code-cell} ipython3 -group_pca = pca(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'], - n_components=3, - whiten=True, - name='grouped') -``` - -```{code-cell} ipython3 -pca_design = ModelSpec([group_pca], intercept=False) -ISLP_features = pca_design.fit_transform(Carseats) -ISLP_features.columns -``` - -## Direct comparison - -```{code-cell} ipython3 -X = np.asarray(Carseats[['CompPrice', 'Income', 'Advertising', 'Population', 'Price']]) -sklearn_features = sklearn_pca.fit_transform(X) -``` - -```{code-cell} ipython3 -np.linalg.norm(ISLP_features - sklearn_features), np.linalg.norm(ISLP_features - np.asarray(derived_features)) -``` diff --git a/docs/jupyterbook/transforms/poly.ipynb b/docs/jupyterbook/transforms/poly.ipynb deleted file mode 100644 index 45c862e..0000000 --- a/docs/jupyterbook/transforms/poly.ipynb +++ /dev/null @@ -1,341 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "69c2a17b", - "metadata": {}, - "source": [ - "# Polynomial features\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of orthogonal polynomials of features.\n", - "\n", - "Force rebuild" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "4651f149", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from ISLP import load_data\n", - "from ISLP.models import ModelSpec, poly" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "290af7e0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "1b68208a-218c-4ee9-a7d2-8a0b0451b03b", - "metadata": {}, - "source": [ - "Let's make a term representing a quartic effect for `Population`." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8c85fa88-cd27-4a83-9060-dd8dc178c83f", - "metadata": {}, - "outputs": [], - "source": [ - "quartic = poly('Population', 4)" - ] - }, - { - "cell_type": "markdown", - "id": "8e1df9d6-b2d3-42aa-addf-1155b30973e5", - "metadata": {}, - "source": [ - "The object `quartic` does not refer to any data yet, it must be included in a `ModelSpec` object\n", - "and fit using the `fit` method." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a49ad3e4-af07-4952-9a4a-590fca565d2c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['poly(Population, degree=4)[0]', 'poly(Population, degree=4)[1]',\n", - " 'poly(Population, degree=4)[2]', 'poly(Population, degree=4)[3]'],\n", - " dtype='object')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "design = ModelSpec([quartic], intercept=False)\n", - "ISLP_features = design.fit_transform(Carseats)\n", - "ISLP_features.columns" - ] - }, - { - "cell_type": "markdown", - "id": "9df16fd2-6427-4f1e-97d8-1d3e28ff1f0f", - "metadata": {}, - "source": [ - "## Compare to `R`\n", - "\n", - "We can compare our polynomials to a similar function in `R`" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "d6fde88f-cde3-43f1-8814-36ce4a4b4ecc", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext rpy2.ipython" - ] - }, - { - "cell_type": "markdown", - "id": "a62ef797-8c9c-45e4-a08d-b2efa8df709e", - "metadata": {}, - "source": [ - "We'll recompute these features using `poly` in `R`." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "1bc583ca-6325-4419-8ef5-cc42f970b67a", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats -o R_features\n", - "R_features = poly(Carseats$Population, 4)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "052f2e9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3.95855733196009e-15" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(ISLP_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "c4e1679d-6434-45e3-a675-33c1ad183f04", - "metadata": {}, - "source": [ - "## Underlying model\n", - "\n", - "If we look at `quartic`, we see it is a `Feature`, i.e. it can be used to produce a set of columns\n", - "in a design matrix when it is a term used in creating the `ModelSpec`.\n", - "\n", - "Its encoder is `Poly(degree=4)`. This is a special `sklearn` transform that expects a single column\n", - "in its `fit()` method and constructs a matrix of corresponding orthogonal polynomials.\n", - "\n", - "The spline helpers `ns` and `bs` as well as `pca` follow a similar structure." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "864c8a30-167c-40f8-a5b1-b26ba158aa21", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable(variables=('Population',), name='poly(Population, degree=4)', encoder=Poly(degree=4), use_transform=True, pure_columns=False, override_encoder_colnames=True)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "quartic" - ] - }, - { - "cell_type": "markdown", - "id": "9c31a4e6-11f2-4d90-9112-25756f88cf24", - "metadata": {}, - "source": [ - "## Raw polynomials\n", - "\n", - "One can compute raw polynomials (which results in a less well-conditioned design matrix) of course." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "c348c5b6-d542-48bc-8da0-cdebf404d331", - "metadata": {}, - "outputs": [], - "source": [ - "quartic_raw = poly('Population', degree=4, raw=True)" - ] - }, - { - "cell_type": "markdown", - "id": "8eea8b00-ed2b-4cdf-90dc-bf1d7406733d", - "metadata": {}, - "source": [ - "Let's compare the features again." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "07053253-7d7b-474a-a319-7982e052fb75", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([quartic_raw], intercept=False)\n", - "raw_features = design.fit_transform(Carseats)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "986583bb-b819-4d1e-bc4a-1288f94c16ab", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats -o R_features\n", - "R_features = poly(Carseats$Population, 4, raw=TRUE)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "00abce93-42ef-4f28-b9f7-8e1c7e24b3a5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(raw_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "2097fc9b-62ae-4ee7-aea4-0d65f9a07db3", - "metadata": {}, - "source": [ - "## Intercept\n", - "\n", - "Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, \n", - "it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True`" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "a2d6b36c-29ff-4d5e-ae1e-a7cab4bc68b4", - "metadata": {}, - "outputs": [], - "source": [ - "quartic_int = poly('Population', degree=4, raw=True, intercept=True)\n", - "design = ModelSpec([quartic_int], intercept=False)\n", - "intercept_features = design.fit_transform(Carseats)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "1ef711fd-5be2-4267-9a79-1553e4da91e9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(intercept_features.iloc[:,1:] - R_features)" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyterbook/transforms/poly.md b/docs/jupyterbook/transforms/poly.md deleted file mode 100644 index e5aef11..0000000 --- a/docs/jupyterbook/transforms/poly.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -jupytext: - formats: source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.14.5 -kernelspec: - display_name: python3 - language: python - name: python3 ---- - -# Polynomial features - -The modelling tools included in `ISLP` allow for -construction of orthogonal polynomials of features. - -Force rebuild - -```{code-cell} ipython3 -import numpy as np -from ISLP import load_data -from ISLP.models import ModelSpec, poly -``` - -```{code-cell} ipython3 -Carseats = load_data('Carseats') -Carseats.columns -``` - -Let's make a term representing a quartic effect for `Population`. - -```{code-cell} ipython3 -quartic = poly('Population', 4) -``` - -The object `quartic` does not refer to any data yet, it must be included in a `ModelSpec` object -and fit using the `fit` method. - -```{code-cell} ipython3 -design = ModelSpec([quartic], intercept=False) -ISLP_features = design.fit_transform(Carseats) -ISLP_features.columns -``` - -## Compare to `R` - -We can compare our polynomials to a similar function in `R` - -```{code-cell} ipython3 -%load_ext rpy2.ipython -``` - -We'll recompute these features using `poly` in `R`. - -```{code-cell} ipython3 -%%R -i Carseats -o R_features -R_features = poly(Carseats$Population, 4) -``` - -```{code-cell} ipython3 -np.linalg.norm(ISLP_features - R_features) -``` - -## Underlying model - -If we look at `quartic`, we see it is a `Feature`, i.e. it can be used to produce a set of columns -in a design matrix when it is a term used in creating the `ModelSpec`. - -Its encoder is `Poly(degree=4)`. This is a special `sklearn` transform that expects a single column -in its `fit()` method and constructs a matrix of corresponding orthogonal polynomials. - -The spline helpers `ns` and `bs` as well as `pca` follow a similar structure. - -```{code-cell} ipython3 -quartic -``` - -## Raw polynomials - -One can compute raw polynomials (which results in a less well-conditioned design matrix) of course. - -```{code-cell} ipython3 -quartic_raw = poly('Population', degree=4, raw=True) -``` - -Let's compare the features again. - -```{code-cell} ipython3 -design = ModelSpec([quartic_raw], intercept=False) -raw_features = design.fit_transform(Carseats) -``` - -```{code-cell} ipython3 -%%R -i Carseats -o R_features -R_features = poly(Carseats$Population, 4, raw=TRUE) -``` - -```{code-cell} ipython3 -np.linalg.norm(raw_features - R_features) -``` - -## Intercept - -Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, -it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True` - -```{code-cell} ipython3 -quartic_int = poly('Population', degree=4, raw=True, intercept=True) -design = ModelSpec([quartic_int], intercept=False) -intercept_features = design.fit_transform(Carseats) -``` - -```{code-cell} ipython3 -np.linalg.norm(intercept_features.iloc[:,1:] - R_features) -``` diff --git a/docs/jupyterbook/transforms/splines.ipynb b/docs/jupyterbook/transforms/splines.ipynb deleted file mode 100644 index 399b0be..0000000 --- a/docs/jupyterbook/transforms/splines.ipynb +++ /dev/null @@ -1,332 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4a50ef04", - "metadata": {}, - "source": [ - "# Spline features\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of spline functions of features.\n", - "\n", - "Force rebuild" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "666e6a04", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from ISLP import load_data\n", - "from ISLP.models import ModelSpec, ns, bs" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "0ecda36e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "20c6e551", - "metadata": {}, - "source": [ - "Let's make a term representing a cubic spline for `Population`. We'll use knots based on the \n", - "deciles." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "31f3f4ff-a63f-4374-b484-fbd3b27ce1ad", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 58.9, 110.4, 160. , 218.6, 272. , 317.8, 366. , 412.2, 467. ])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "knots = np.percentile(Carseats['Population'], np.linspace(10, 90, 9))\n", - "knots" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1ea79e0d", - "metadata": {}, - "outputs": [], - "source": [ - "bs_pop = bs('Population', internal_knots=knots, degree=3)" - ] - }, - { - "cell_type": "markdown", - "id": "a47d2caa", - "metadata": {}, - "source": [ - "The object `bs_pop` does not refer to any data yet, it must be included in a `ModelSpec` object\n", - "and fit using the `fit` method." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "90e8f810", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([bs_pop], intercept=False)\n", - "py_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "markdown", - "id": "f98f658a", - "metadata": {}, - "source": [ - "## Compare to `R`\n", - "\n", - "We can compare our polynomials to a similar function in `R`" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "fda53527", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext rpy2.ipython" - ] - }, - { - "cell_type": "markdown", - "id": "20c76684", - "metadata": {}, - "source": [ - "We'll recompute these features using `bs` in `R`. The default knot selection of the\n", - "`ISLP` and `R` version are slightly different so we just fix the set of internal knots." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "5d5e7985", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats,knots -o R_features\n", - "library(splines)\n", - "R_features = bs(Carseats$Population, knots=knots, degree=3)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "ceffa80f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1.1372379284497324e-15" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(py_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "a89ec276-cbf5-4fed-9720-c33060570fd9", - "metadata": {}, - "source": [ - "## Underlying model\n", - "\n", - "As for `poly`, the computation of the B-splines is done by a special `sklearn` transformer." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "cacf2134-70ad-48d7-aa1c-83c2c88d05f9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable(variables=('Population',), name='bs(Population, internal_knots=[ 58.9 110.4 160. 218.6 272. 317.8 366. 412.2 467. ], degree=3)', encoder=BSpline(internal_knots=array([ 58.9, 110.4, 160. , 218.6, 272. , 317.8, 366. , 412.2, 467. ]),\n", - " lower_bound=10.0, upper_bound=509.0), use_transform=True, pure_columns=False, override_encoder_colnames=True)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bs_pop" - ] - }, - { - "cell_type": "markdown", - "id": "7bd02961-3f17-4352-af69-c8229656ae96", - "metadata": {}, - "source": [ - "## Natural splines \n", - "\n", - "Natural cubic splines are also implemented." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "3c7c2445", - "metadata": {}, - "outputs": [], - "source": [ - "ns_pop = ns('Population', internal_knots=knots)\n", - "design = ModelSpec([ns_pop], intercept=False)\n", - "py_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "805060ca-5aac-46b4-842e-bbf4f39dc429", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -o R_features\n", - "library(splines)\n", - "R_features = ns(Carseats$Population, knots=knots)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "6fb40058-6a80-431a-8c45-448d742d5947", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1.2473757226554746e-15" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(py_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "39bc9180", - "metadata": {}, - "source": [ - "## Intercept\n", - "\n", - "Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, \n", - "it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True`. This means that the\n", - "column space includes an intercept, though there is no specific column labeled as intercept." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "38de6cf7", - "metadata": {}, - "outputs": [], - "source": [ - "bs_int = ns('Population', internal_knots=knots, intercept=True)\n", - "design = ModelSpec([bs_int], intercept=False)\n", - "py_int_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "1dcd7ff8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((400, 11), (400, 10))" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "py_int_features.shape, py_features.shape" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyterbook/transforms/splines.md b/docs/jupyterbook/transforms/splines.md deleted file mode 100644 index de0ee3d..0000000 --- a/docs/jupyterbook/transforms/splines.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -jupytext: - formats: source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.14.5 -kernelspec: - display_name: python3 - language: python - name: python3 ---- - -# Spline features - -The modelling tools included in `ISLP` allow for -construction of spline functions of features. - -Force rebuild - -```{code-cell} ipython3 -import numpy as np -from ISLP import load_data -from ISLP.models import ModelSpec, ns, bs -``` - -```{code-cell} ipython3 -Carseats = load_data('Carseats') -Carseats.columns -``` - -Let's make a term representing a cubic spline for `Population`. We'll use knots based on the -deciles. - -```{code-cell} ipython3 -knots = np.percentile(Carseats['Population'], np.linspace(10, 90, 9)) -knots -``` - -```{code-cell} ipython3 -bs_pop = bs('Population', internal_knots=knots, degree=3) -``` - -The object `bs_pop` does not refer to any data yet, it must be included in a `ModelSpec` object -and fit using the `fit` method. - -```{code-cell} ipython3 -design = ModelSpec([bs_pop], intercept=False) -py_features = np.asarray(design.fit_transform(Carseats)) -``` - -## Compare to `R` - -We can compare our polynomials to a similar function in `R` - -```{code-cell} ipython3 -%load_ext rpy2.ipython -``` - -We'll recompute these features using `bs` in `R`. The default knot selection of the -`ISLP` and `R` version are slightly different so we just fix the set of internal knots. - -```{code-cell} ipython3 -%%R -i Carseats,knots -o R_features -library(splines) -R_features = bs(Carseats$Population, knots=knots, degree=3) -``` - -```{code-cell} ipython3 -np.linalg.norm(py_features - R_features) -``` - -## Underlying model - -As for `poly`, the computation of the B-splines is done by a special `sklearn` transformer. - -```{code-cell} ipython3 -bs_pop -``` - -## Natural splines - -Natural cubic splines are also implemented. - -```{code-cell} ipython3 -ns_pop = ns('Population', internal_knots=knots) -design = ModelSpec([ns_pop], intercept=False) -py_features = np.asarray(design.fit_transform(Carseats)) -``` - -```{code-cell} ipython3 -%%R -o R_features -library(splines) -R_features = ns(Carseats$Population, knots=knots) -``` - -```{code-cell} ipython3 -np.linalg.norm(py_features - R_features) -``` - -## Intercept - -Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, -it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True`. This means that the -column space includes an intercept, though there is no specific column labeled as intercept. - -```{code-cell} ipython3 -bs_int = ns('Population', internal_knots=knots, intercept=True) -design = ModelSpec([bs_int], intercept=False) -py_int_features = np.asarray(design.fit_transform(Carseats)) -``` - -```{code-cell} ipython3 -py_int_features.shape, py_features.shape -``` diff --git a/docs/source/api/gen.rst b/docs/source/api/gen.rst index fb3bec5..529ab84 100644 --- a/docs/source/api/gen.rst +++ b/docs/source/api/gen.rst @@ -9,9 +9,7 @@ generated/ISLP.bart.tree generated/ISLP.cluster generated/ISLP.models - generated/ISLP.models.columns generated/ISLP.models.generic_selector - generated/ISLP.models.model_spec generated/ISLP.models.sklearn_wrap generated/ISLP.models.strategy generated/ISLP.pygam @@ -20,4 +18,3 @@ generated/ISLP.torch generated/ISLP.torch.imdb generated/ISLP.torch.lightning - generated/ISLP.transforms diff --git a/docs/source/api/generated/ISLP.models.columns.rst b/docs/source/api/generated/ISLP.models.columns.rst deleted file mode 100644 index 0641cb3..0000000 --- a/docs/source/api/generated/ISLP.models.columns.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. AUTO-GENERATED FILE -- DO NOT EDIT! - -models.columns -============== - -Module: :mod:`models.columns` ------------------------------ -Inheritance diagram for ``ISLP.models.columns``: - -.. inheritance-diagram:: ISLP.models.columns - :parts: 3 - -.. automodule:: ISLP.models.columns - -.. currentmodule:: ISLP.models.columns - -:class:`Column` ---------------- - - -.. autoclass:: Column - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ diff --git a/docs/source/api/generated/ISLP.models.model_spec.rst b/docs/source/api/generated/ISLP.models.model_spec.rst deleted file mode 100644 index d457e3a..0000000 --- a/docs/source/api/generated/ISLP.models.model_spec.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. AUTO-GENERATED FILE -- DO NOT EDIT! - -models.model_spec -================= - -Module: :mod:`models.model_spec` --------------------------------- -Inheritance diagram for ``ISLP.models.model_spec``: - -.. inheritance-diagram:: ISLP.models.model_spec - :parts: 3 - -.. automodule:: ISLP.models.model_spec - -.. currentmodule:: ISLP.models.model_spec - -Classes -------- - -:class:`Contrast` -~~~~~~~~~~~~~~~~~ - - -.. autoclass:: Contrast - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -:class:`Feature` -~~~~~~~~~~~~~~~~ - - -.. autoclass:: Feature - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -:class:`ModelSpec` -~~~~~~~~~~~~~~~~~~ - - -.. autoclass:: ModelSpec - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -Functions ---------- - - -.. autofunction:: ISLP.models.model_spec.bs - - -.. autofunction:: ISLP.models.model_spec.build_columns - - -.. autofunction:: ISLP.models.model_spec.build_model - - -.. autofunction:: ISLP.models.model_spec.contrast - - -.. autofunction:: ISLP.models.model_spec.derived_feature - - -.. autofunction:: ISLP.models.model_spec.fit_encoder - - -.. autofunction:: ISLP.models.model_spec.ns - - -.. autofunction:: ISLP.models.model_spec.ordinal - - -.. autofunction:: ISLP.models.model_spec.pca - - -.. autofunction:: ISLP.models.model_spec.poly - diff --git a/docs/source/api/generated/ISLP.transforms.rst b/docs/source/api/generated/ISLP.transforms.rst deleted file mode 100644 index c7a2fd3..0000000 --- a/docs/source/api/generated/ISLP.transforms.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. AUTO-GENERATED FILE -- DO NOT EDIT! - -transforms -========== - -Module: :mod:`transforms` -------------------------- -Inheritance diagram for ``ISLP.transforms``: - -.. inheritance-diagram:: ISLP.transforms - :parts: 3 - -.. automodule:: ISLP.transforms - -.. currentmodule:: ISLP.transforms - -Classes -------- - -:class:`BSpline` -~~~~~~~~~~~~~~~~ - - -.. autoclass:: BSpline - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -:class:`Interaction` -~~~~~~~~~~~~~~~~~~~~ - - -.. autoclass:: Interaction - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -:class:`NaturalSpline` -~~~~~~~~~~~~~~~~~~~~~~ - - -.. autoclass:: NaturalSpline - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ - -:class:`Poly` -~~~~~~~~~~~~~ - - -.. autoclass:: Poly - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - .. automethod:: __init__ diff --git a/docs/source/conf.py b/docs/source/conf.py index 2434f25..6181c09 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,8 +55,7 @@ [f'Ch{i:02d}*' for i in range(2, 14)]) print('exclude patterns', nb_execution_excludepatterns) nb_execution_allow_errors = True -# warnings (stderr), e.g. ModelSpec's FutureWarning that it moves to its own -# package in ISLP 0.7, are not shown in the pages; errors still are +# warnings (stderr) are not shown in the pages; errors still are nb_output_stderr = 'remove' #nb_kernel_rgx_aliases = {'python3': "islp_test"} @@ -67,6 +66,9 @@ 'pytorch_lightning': ('https://pytorch-lightning.readthedocs.io/en/latest/', None), 'torch': ('https://pytorch.org/docs/stable/', None), 'sklearn': ('http://scikit-learn.org/stable', None), + # ModelSpec and the transforms: references such as + # :py:class:`modelmatrix.ModelSpec` link to modelmatrix's API + 'modelmatrix': ('https://intro-stat-learning.github.io/modelmatrix/api/', None), } intersphinx_disabled_domains = ['std'] diff --git a/docs/source/models.rst b/docs/source/models.rst index 95af872..52179d4 100644 --- a/docs/source/models.rst +++ b/docs/source/models.rst @@ -1,12 +1,14 @@ Tools for regression models =========================== +Model matrices are built by :py:class:`~modelmatrix.ModelSpec`, of the +`modelmatrix `_ package, which `ISLP.models` re-exports: +its pages are part of modelmatrix's documentation. + .. toctree:: - models/spec + Building model matrices: ModelSpec models/selection - models/anova - models/comparison - - - + ANOVA + Comparison with R, patsy and formulaic + Responses and the wildcard diff --git a/docs/source/models/anova.ipynb b/docs/source/models/anova.ipynb deleted file mode 100644 index bf77f73..0000000 --- a/docs/source/models/anova.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "ee33d364", - "metadata": {}, - "source": [ - "# ANOVA using `ModelSpec`\n", - "\n", - "\n", - "In this lab we illustrate how to run create specific ANOVA analyses\n", - "using `ModelSpec`." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "4c70fbaa", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from statsmodels.api import OLS\n", - "from statsmodels.stats.anova import anova_lm\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec,\n", - " derived_feature,\n", - " summarize)" - ] - }, - { - "cell_type": "markdown", - "id": "333a49cf", - "metadata": {}, - "source": [ - "We will carry out two simple ANOVA analyses of the `Hitters` data.\n", - "We wish to predict a baseball player’s `Salary` on the\n", - "basis of various statistics associated with performance in the\n", - "previous year." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8a708215", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "59" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Hitters = load_data('Hitters')\n", - "np.isnan(Hitters['Salary']).sum()" - ] - }, - { - "cell_type": "markdown", - "id": "dad5e991", - "metadata": {}, - "source": [ - " \n", - " We see that `Salary` is missing for 59 players. The\n", - "`dropna()` method of data frames removes all of the rows that have missing\n", - "values in any variable (by default --- see `Hitters.dropna?`)." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ac7086a5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years', 'CAtBat',\n", - " 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'League', 'Division',\n", - " 'PutOuts', 'Assists', 'Errors', 'Salary', 'NewLeague'],\n", - " dtype='object')" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Hitters = Hitters.dropna()\n", - "Hitters.columns" - ] - }, - { - "cell_type": "markdown", - "id": "1a0a3521-be74-40df-a404-3895d80a11dc", - "metadata": {}, - "source": [ - "## Grouping variables\n", - "\n", - "A look at the [description](https://islp.readthedocs.io/en/latest/datasets/Hitters.html) of the data shows\n", - "that there are both career and 1986 offensive stats, as well as some defensive stats.\n", - "\n", - "Let's group the offensive into recent and career offensive stats, as well as a group of defensive variables." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a215e43b-7bc8-4bdd-91cf-40d717cd7978", - "metadata": {}, - "outputs": [], - "source": [ - "confounders = derived_feature(['Division', 'League', 'NewLeague'],\n", - " name='confounders')\n", - "offense_career = derived_feature(['CAtBat', 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks'],\n", - " name='offense_career')\n", - "offense_1986 = derived_feature(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks'],\n", - " name='offense_1986')\n", - "defense_1986 = derived_feature(['PutOuts', 'Assists', 'Errors'],\n", - " name='defense_1986')" - ] - }, - { - "cell_type": "markdown", - "id": "aa15fd0c-1e8a-431e-8425-c61da8439976", - "metadata": {}, - "source": [ - "We'll first do a sequential ANOVA where terms are added sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "40cd6c28", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([confounders, offense_career, defense_1986, offense_1986], intercept=True).fit(Hitters)\n", - "Y = np.array(Hitters['Salary'])\n", - "X = design.transform(Hitters)" - ] - }, - { - "cell_type": "markdown", - "id": "074120b1", - "metadata": {}, - "source": [ - "Along with a score we need to specify the search strategy. This is done through the object\n", - "`Stepwise()` in the `ISLP.models` package. The method `Stepwise.first_peak()`\n", - "runs forward stepwise until any further additions to the model do not result\n", - "in an improvement in the evaluation score. Similarly, the method `Stepwise.fixed_steps()`\n", - "runs a fixed number of steps of stepwise search." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "e65f5607", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
coefstd errtP>|t|
intercept148.218773.5952.0140.045
Division[W]-116.040440.188-2.8870.004
League[N]63.750379.0060.8070.421
NewLeague[N]-24.398978.843-0.3090.757
CAtBat-0.18870.120-1.5720.117
CHits0.16360.6650.2460.806
CHmRun-0.15171.612-0.0940.925
CRuns1.47160.7471.9710.050
CRBI0.80210.6911.1610.247
CWalks-0.81240.327-2.4810.014
PutOuts0.28270.0773.6610.000
Assists0.37550.2201.7050.089
Errors-3.29404.377-0.7530.452
AtBat-1.95090.624-3.1250.002
Hits7.43952.3633.1480.002
HmRun4.34496.1900.7020.483
Runs-2.33122.971-0.7850.433
RBI-1.06702.595-0.4110.681
Walks6.21961.8253.4090.001
\n", - "
" - ], - "text/plain": [ - " coef std err t P>|t|\n", - "intercept 148.2187 73.595 2.014 0.045\n", - "Division[W] -116.0404 40.188 -2.887 0.004\n", - "League[N] 63.7503 79.006 0.807 0.421\n", - "NewLeague[N] -24.3989 78.843 -0.309 0.757\n", - "CAtBat -0.1887 0.120 -1.572 0.117\n", - "CHits 0.1636 0.665 0.246 0.806\n", - "CHmRun -0.1517 1.612 -0.094 0.925\n", - "CRuns 1.4716 0.747 1.971 0.050\n", - "CRBI 0.8021 0.691 1.161 0.247\n", - "CWalks -0.8124 0.327 -2.481 0.014\n", - "PutOuts 0.2827 0.077 3.661 0.000\n", - "Assists 0.3755 0.220 1.705 0.089\n", - "Errors -3.2940 4.377 -0.753 0.452\n", - "AtBat -1.9509 0.624 -3.125 0.002\n", - "Hits 7.4395 2.363 3.148 0.002\n", - "HmRun 4.3449 6.190 0.702 0.483\n", - "Runs -2.3312 2.971 -0.785 0.433\n", - "RBI -1.0670 2.595 -0.411 0.681\n", - "Walks 6.2196 1.825 3.409 0.001" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "M = OLS(Y, X).fit()\n", - "summarize(M)" - ] - }, - { - "cell_type": "markdown", - "id": "29d9b55f", - "metadata": {}, - "source": [ - "We'll first produce the sequential, or Type I ANOVA results. This builds up a model sequentially and compares\n", - "two successive models." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "cfbe5b92", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
df_residssrdf_diffss_diffFPr(>F)
intercept262.05.331911e+070.0NaNNaNNaN
confounders259.05.131263e+073.02.006478e+066.7411472.144265e-04
offense_career253.03.059130e+076.02.072134e+0734.8086561.470455e-30
defense_1986250.02.730614e+073.03.285156e+0611.0371117.880207e-07
offense_1986244.02.420857e+076.03.097572e+065.2034444.648586e-05
\n", - "
" - ], - "text/plain": [ - " df_resid ssr df_diff ss_diff F \\\n", - "intercept 262.0 5.331911e+07 0.0 NaN NaN \n", - "confounders 259.0 5.131263e+07 3.0 2.006478e+06 6.741147 \n", - "offense_career 253.0 3.059130e+07 6.0 2.072134e+07 34.808656 \n", - "defense_1986 250.0 2.730614e+07 3.0 3.285156e+06 11.037111 \n", - "offense_1986 244.0 2.420857e+07 6.0 3.097572e+06 5.203444 \n", - "\n", - " Pr(>F) \n", - "intercept NaN \n", - "confounders 2.144265e-04 \n", - "offense_career 1.470455e-30 \n", - "defense_1986 7.880207e-07 \n", - "offense_1986 4.648586e-05 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = anova_lm(*[OLS(Y, D).fit() for D in design.build_sequence(Hitters, anova_type='sequential')])\n", - "df.index = design.names\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "7092f666", - "metadata": {}, - "source": [ - "We can similarly compute the Type II ANOVA results which drops each term and compares to the full model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "e2d43844", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
df_residssrdf_diffss_diffFPr(>F)
intercept244.02.420857e+071.04.024254e+054.0560764.511037e-02
confounders244.02.420857e+073.09.661738e+053.2460462.261572e-02
offense_career244.02.420857e+076.01.051025e+0717.6555965.701196e-17
defense_1986244.02.420857e+073.01.467933e+064.9318032.415732e-03
offense_1986244.02.420857e+076.03.097572e+065.2034444.648586e-05
\n", - "
" - ], - "text/plain": [ - " df_resid ssr df_diff ss_diff F \\\n", - "intercept 244.0 2.420857e+07 1.0 4.024254e+05 4.056076 \n", - "confounders 244.0 2.420857e+07 3.0 9.661738e+05 3.246046 \n", - "offense_career 244.0 2.420857e+07 6.0 1.051025e+07 17.655596 \n", - "defense_1986 244.0 2.420857e+07 3.0 1.467933e+06 4.931803 \n", - "offense_1986 244.0 2.420857e+07 6.0 3.097572e+06 5.203444 \n", - "\n", - " Pr(>F) \n", - "intercept 4.511037e-02 \n", - "confounders 2.261572e-02 \n", - "offense_career 5.701196e-17 \n", - "defense_1986 2.415732e-03 \n", - "offense_1986 4.648586e-05 " - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "D_full = design.transform(Hitters)\n", - "OLS_full = OLS(Y, D_full).fit()\n", - "dfs = []\n", - "for d in design.build_sequence(Hitters, anova_type='drop'):\n", - " dfs.append(anova_lm(OLS(Y,d).fit(), OLS_full).iloc[1:])\n", - "df = pd.concat(dfs)\n", - "df.index = design.names\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "362709ae-9558-4c4c-8f5e-f8388caf631d", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "jupytext": { - "formats": "source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/models/comparison.myst b/docs/source/models/comparison.myst deleted file mode 100644 index e0eea34..0000000 --- a/docs/source/models/comparison.myst +++ /dev/null @@ -1,459 +0,0 @@ ---- -file_format: mystnb -kernelspec: - name: python3 - display_name: python3 ---- - -# ModelSpec, ColumnTransformer, patsy and formulaic - -`ModelSpec` builds model matrices, as do `sklearn`'s -[ColumnTransformer](https://scikit-learn.org/stable/modules/generated/sklearn.compose.ColumnTransformer.html), -[patsy](https://patsy.readthedocs.io) and -[formulaic](https://matthewwardrop.github.io/formulaic/). The two -formula libraries parse a string such as `'Sales ~ Price * ShelveLoc'` -in the spirit of R's model formulae. A `ColumnTransformer` lists a -transformer and the columns it applies to, for each block of columns. -`ModelSpec` takes a list of Python objects: column names, tuples for -interactions, and *features* that wrap any `sklearn` transformer. - -This page compares them on flexibility and extensibility: what it -takes to add a new kind of term, to nest one feature inside another, -to interact terms, and to work inside `sklearn`. Reproducing R's -formula semantics is not a goal of `ModelSpec`, so some defaults -differ on purpose; those are collected at the end. - -## At a glance - -| | `ModelSpec` | `ColumnTransformer` | patsy | formulaic | -|---|---|---|---|---| -| any `sklearn` transformer as a term | directly | directly | with a stateful-transform wrapper | with a stateful-transform wrapper | -| interacting terms (e.g. a spline with a categorical) | a tuple | no: `PolynomialFeatures` multiplies every pair of columns | `*` and `:` | `*` and `:` | -| a feature of other features (e.g. PCA of a polynomial and dummies) | yes | yes, nesting `Pipeline`s and `ColumnTransformer`s | no | no | -| categorical columns | coded from their dtype | an encoder you name, for columns you list or select by dtype | coded from their dtype | coded from their dtype | -| a model fitted elsewhere as a term | yes, with `FrozenEstimator` | with a `FunctionTransformer` around it | yes, a function calling it | yes, a function calling it | -| building a specification | Python lists of objects | lists of (name, transformer, columns) | formula strings | formula strings | -| `sklearn` transformer (`Pipeline`, refit in each CV fold) | yes | yes | needs a wrapper estimator | needs a wrapper estimator | -| tuning a term's parameters by `GridSearchCV` | yes (`modelspec__terms__0__df`) | yes (`columntransformer__spline__n_knots`) | needs a wrapper estimator | needs a wrapper estimator | -| pickling a fitted specification | yes | yes | no | yes | -| sparse model matrices | yes (`sparse_output=True`) | yes (`sparse_threshold`) | no | yes (`output='sparse'`) | -| input other than `pandas` (e.g. polars) | yes; output in the same backend | yes; output by `set_output` | no | yes | - -`ModelSpec` warns, when fit, that it moves to its own package in -ISLP 0.7; the warning is silenced here. - -```{code-cell} ipython3 -import pickle, warnings -warnings.filterwarnings('ignore', message='ModelSpec will move') -import numpy as np, pandas as pd -import patsy, formulaic -import sklearn - -from ISLP import load_data -from ISLP import models as islp -from ISLP.models import ModelSpec as MS - -from sklearn.base import clone -from sklearn.decomposition import PCA -from sklearn.preprocessing import SplineTransformer -from sklearn.pipeline import make_pipeline -from sklearn.linear_model import LinearRegression -from sklearn.neighbors import KNeighborsRegressor -from sklearn.model_selection import cross_val_score, GridSearchCV - -Carseats = load_data('Carseats') -new = Carseats.iloc[:5] -print(f'patsy {patsy.__version__}, formulaic {formulaic.__version__}, ' - f'scikit-learn {sklearn.__version__}, pandas {pd.__version__}') -``` - -The `ISLP` helpers are accessed as `islp.poly`, `islp.ns`, etc. rather -than imported by name. Formula strings are evaluated in the calling -namespace, so importing `ISLP`'s `bs` would silently replace patsy's -`bs` inside `patsy.dmatrix('bs(Price, df=5)', ...)`. - -## Terms are Python objects - -Any `sklearn` transformer can be used as a term with `derived_feature`, -with no registration or wrapper. Its state is fit when the `ModelSpec` -is fit, and reused on new data. - -```{code-cell} ipython3 -spline = islp.derived_feature(['Price'], - encoder=SplineTransformer(n_knots=5), - name='spline(Price)') -design = MS([spline, 'ShelveLoc'], intercept=True).fit(Carseats) -design.transform(new) -``` - -Because terms are objects, a list of them can be built with ordinary -Python, for example a natural spline for every numeric column: - -```{code-cell} ipython3 -numeric = [c for c in Carseats.columns.drop('Sales') - if pd.api.types.is_numeric_dtype(Carseats[c])] -design = MS([islp.ns(c, df=3) for c in numeric] + ['ShelveLoc', 'US'], - intercept=True) -design.fit_transform(Carseats).shape -``` - -The inputs of a feature may themselves be features, which are -encoded first. Here a PCA is fit to an orthogonal polynomial in -`Price`, `Income` and the one-hot encoding of `ShelveLoc`, and its -scores are interacted with `US`: - -```{code-cell} ipython3 -scores = islp.derived_feature([islp.poly('Price', 2), - 'Income', - islp.contrast('ShelveLoc', None)], - encoder=PCA(n_components=2), - name='pc') -design = MS([scores, 'US', (scores, 'US')], intercept=True).fit(Carseats) -design.transform(new) -``` - -Fitting a `ModelSpec` fits copies of the encoders of its terms, so the -same terms can be used in several specifications, and a fitted -specification can be fit again. A model fit elsewhere, on other data, -can be a term too: wrapped in `sklearn`'s `FrozenEstimator`, it is used -as is and never refit. Here a nearest-neighbour fit to the first 200 -stores supplies a column for the others: - -```{code-cell} ipython3 -from sklearn.frozen import FrozenEstimator - -first, rest = Carseats.iloc[:200], Carseats.iloc[200:] -knn = KNeighborsRegressor(10).fit(first[['Price', 'Income']], first['Sales']) -knn_term = islp.derived_feature(['Price', 'Income'], - encoder=FrozenEstimator(knn), - name='knn', - use_transform=False) # its predictions -MS([knn_term, 'ShelveLoc'], intercept=True).fit_transform(rest).head() -``` - -## The same in patsy and formulaic - -Both formula libraries support stateful transforms, but an `sklearn` -transformer needs a wrapper written against each library's protocol. - -```{code-cell} ipython3 -from formulaic.transforms import stateful_transform - -class _PatsySklearn: - # patsy's stateful transform protocol, wrapping an sklearn transformer - def __init__(self): - self.chunks = [] - def memorize_chunk(self, *cols, est): - self.est = est - self.chunks.append(np.column_stack(cols)) - def memorize_finish(self): - self.est_ = clone(self.est).fit(np.vstack(self.chunks)) - def transform(self, *cols, est): - return self.est_.transform(np.column_stack(cols)) - -patsy_sk = patsy.stateful_transform(_PatsySklearn) - -@stateful_transform -def formulaic_sk(*cols, est, _state=None): - X = np.column_stack(cols) - if 'est' not in _state: - _state['est'] = clone(est).fit(X) - return _state['est'].transform(X) - -spl = SplineTransformer(n_knots=5) -X_patsy = patsy.dmatrix('patsy_sk(Price, est=spl) + ShelveLoc', Carseats) -X_formulaic = formulaic.model_matrix('formulaic_sk(Price, est=spl) + ShelveLoc', Carseats) -X_ms = MS([spline, 'ShelveLoc'], intercept=True).fit_transform(Carseats) -X_patsy.shape, X_formulaic.shape, X_ms.shape -``` - -With the wrapper, a learned feature can be interacted with a -categorical in either library. Nesting is a different matter: inside a -function call, `C(ShelveLoc)` is a box holding the labels rather than -its dummy columns, so the PCA above cannot be written this way. - -```{code-cell} ipython3 -pc = PCA(n_components=2) -for name, build in [('patsy', lambda: patsy.dmatrix( - 'patsy_sk(Price, Price**2, Income, C(ShelveLoc), est=pc)', Carseats)), - ('formulaic', lambda: formulaic.model_matrix( - 'formulaic_sk(Price, Price**2, Income, C(ShelveLoc), est=pc)', Carseats))]: - try: - build() - except Exception as e: - print(f'{name}: {type(e).__name__}') -``` - -## Working inside scikit-learn - -A `ModelSpec` is an `sklearn` transformer, so it can be the first step -of a `Pipeline`, and refit on each fold of a cross-validation. Here the -regression fits its own intercept, so the model matrix has none: - -```{code-cell} ipython3 -pipe = make_pipeline(MS(['Price', 'ShelveLoc', 'Income'], intercept=False), - LinearRegression()) -cross_val_score(pipe, Carseats, Carseats['Sales'], cv=5).round(3) -``` - -### Tuning the parameters of terms - -The parameters of each term are parameters of the `ModelSpec`, so -`GridSearchCV` can tune them as it tunes those of any `sklearn` -estimator. A term's parameter is named by the position of the term in -`terms`: - -| key | the parameter of | -|---|---| -| `'terms____'` | the term at position `i`, e.g. `'terms__0__df'` | -| `'terms______'` | part `j` of the interaction at position `i` | -| `'terms____'` | the term given that name, e.g. `ns('Price', df=4, name='price')` | - -`get_params()` lists them: - -```{code-cell} ipython3 -spec = MS([islp.ns('Price', df=4), - 'US', - ('ShelveLoc', islp.poly('Income', 2))], intercept=False) -{k: v for k, v in spec.get_params().items() - if k in ('terms__0__df', 'terms__2__1__degree')} -``` - -In a `Pipeline` each key is prefixed by the name of its step, here -`'modelspec__'`. Both parameters are tuned below. A term made by -`ns()`, `bs()`, `poly()` or `pca()` is made again with its new -parameters, so its name, and the names of its columns, follow: - -```{code-cell} ipython3 -pipe = make_pipeline(spec, LinearRegression()) -grid = {'modelspec__terms__0__df': [1, 2, 4, 8], - 'modelspec__terms__2__1__degree': [1, 2, 3]} -search = GridSearchCV(pipe, grid, cv=5).fit(Carseats, Carseats['Sales']) -best = search.best_estimator_[0] -search.best_params_, best.terms[0].name, best.terms[2][1].name -``` - -The parameters of other terms, e.g. a `derived_feature` of any -transformer or `contrast()`, are those of their encoder -(`'terms____n_knots'` for a `SplineTransformer`, -`'terms____method'` for `contrast()`). Setting a parameter does not -change `terms`: it is replaced by a copy, in which a term used in -several places, e.g. also in an interaction, is changed in all of them. - -So can `terms` itself, to choose among specifications: - -```{code-cell} ipython3 -grid = {'modelspec__terms': [['Price', 'ShelveLoc'], - ['Price', 'ShelveLoc', ('Price', 'ShelveLoc')]]} -search = GridSearchCV(make_pipeline(MS(intercept=False), LinearRegression()), - grid, cv=5) -search.fit(Carseats, Carseats['Sales']) -len(search.best_params_['modelspec__terms']) -``` - -`get_feature_names_out` names the columns, which also enables -`set_output`, here giving a data frame from an `np.ndarray`: - -```{code-cell} ipython3 -A = Carseats[['Price', 'Income']].to_numpy() -(MS([0, islp.poly(1, 2)], intercept=True) - .set_output(transform='pandas') - .fit_transform(A) - .head()) -``` - -## Compared with scikit-learn's ColumnTransformer - -A `ColumnTransformer` fits as naturally into `sklearn` as a `ModelSpec`: -it is a transformer, and the parameters of its transformers can be -tuned. The same model matrix, a spline in `Price`, `ShelveLoc` and -`Income`, is written below both ways; they give the same columns, and -tuning the number of knots of the spline gives the same results. - -```{code-cell} ipython3 -from sklearn.compose import ColumnTransformer, make_column_selector -from sklearn.preprocessing import OneHotEncoder, PolynomialFeatures - -spline = islp.derived_feature(['Price'], encoder=SplineTransformer(n_knots=5), - name='spline') -ms = MS([spline, 'ShelveLoc', 'Income'], intercept=False) - -ct = ColumnTransformer([ - ('spline', SplineTransformer(n_knots=5), ['Price']), - ('shelve', OneHotEncoder(drop='first', sparse_output=False), ['ShelveLoc']), - ('income', 'passthrough', ['Income'])]) - -searches = { - 'ModelSpec': GridSearchCV(make_pipeline(ms, LinearRegression()), - {'modelspec__terms__0__n_knots': [3, 5, 8]}, cv=5), - 'ColumnTransformer': GridSearchCV(make_pipeline(ct, LinearRegression()), - {'columntransformer__spline__n_knots': [3, 5, 8]}, cv=5)} -for name, search in searches.items(): - search.fit(Carseats, Carseats['Sales']) - print(f'{name:18s}', search.best_params_, search.cv_results_['mean_test_score'].round(4)) -``` - -The categorical `ShelveLoc` needs its own encoder in the -`ColumnTransformer`, chosen by name or, as here, by dtype; a -`ModelSpec` codes it from its dtype. - -```{code-cell} ipython3 -ct_by_dtype = ColumnTransformer([ - ('spline', SplineTransformer(n_knots=5), ['Price']), - ('categorical', OneHotEncoder(drop='first', sparse_output=False), - make_column_selector(dtype_include='category')), - ('income', 'passthrough', ['Income'])]) -ct_by_dtype.fit_transform(Carseats[['Price', 'ShelveLoc', 'Income']]).shape -``` - -Interactions are where the two differ most. In a `ModelSpec` a tuple -interacts any terms, here the spline with `ShelveLoc`. A -`ColumnTransformer` has no way to multiply one block of columns by -another: `PolynomialFeatures(interaction_only=True)` multiplies every -pair of its input columns, including pairs of spline columns and of -dummy columns. - -```{code-cell} ipython3 -ms_interaction = MS([spline, 'ShelveLoc', (spline, 'ShelveLoc')], intercept=False) -ct_interaction = make_pipeline( - ColumnTransformer([('spline', SplineTransformer(n_knots=5), ['Price']), - ('shelve', OneHotEncoder(drop='first', sparse_output=False), ['ShelveLoc'])]), - PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)) -print('ModelSpec ', ms_interaction.fit_transform(Carseats).shape[1], 'columns') -print('ColumnTransformer', ct_interaction.fit_transform(Carseats).shape[1], 'columns') -``` - -A fitted `ModelSpec` can be pickled, with all of its encoders, as can -formulaic's `ModelSpec`; patsy's `DesignInfo` cannot. - -```{code-cell} ipython3 -fitted = {'ModelSpec': lambda: MS(['Price', 'ShelveLoc', islp.bs('Price', df=5)], - intercept=True).fit(Carseats), - 'patsy': lambda: patsy.dmatrix('Price + ShelveLoc + bs(Price, df=5)', Carseats).design_info, - 'formulaic': lambda: formulaic.model_matrix('Price + ShelveLoc + bs(Price, df=5)', Carseats).model_spec} -for name, spec in fitted.items(): - try: - pickle.dumps(spec()) - print(f'{name}: pickled') - except Exception as e: - print(f'{name}: {type(e).__name__}') -``` - -## Sparse model matrices - -With `sparse_output=True`, a `ModelSpec` returns a -`scipy.sparse.csr_matrix`. Categorical encodings and their interactions -are built sparse throughout, so a categorical with many levels never -forms its dense columns; formulaic's `output='sparse'` does the same. -Here a categorical with 5,000 levels and 50,000 rows, whose dense model -matrix would take 2 GB: - -```{code-cell} ipython3 -rng = np.random.default_rng(0) -n, K = 50_000, 5_000 -B = pd.DataFrame({'x': rng.standard_normal(n), - 'id': pd.Categorical(rng.integers(0, K, n).astype(str))}) -X_ms = MS(['x', 'id', ('id', 'x')], intercept=True, - sparse_output=True).fit_transform(B) -X_formulaic = formulaic.model_matrix('x + id + id:x', B, output='sparse') -print('ModelSpec', type(X_ms).__name__, X_ms.shape, f'{X_ms.nnz:,} nonzero') -print('formulaic', type(X_formulaic).__name__, X_formulaic.shape, f'{X_formulaic.nnz:,} nonzero') -``` - -## Other dataframes - -Dataframes of the backends supported by -[narwhals](https://narwhals-dev.github.io/narwhals/), such as polars or -pyarrow, can be given to a `ModelSpec`, and the model matrix is returned -in the same backend. A polars `Enum` carries the order of its levels, -so `ordinal()` can score it. formulaic accepts them too; patsy expects -`pandas` or `numpy`. - -```{code-cell} ipython3 -import polars as pl - -P = pl.from_pandas(Carseats).with_columns( - pl.col('ShelveLoc').cast(pl.String).cast(pl.Enum(['Bad', 'Medium', 'Good']))) -design = MS(['Price', 'US', islp.ordinal('ShelveLoc')], intercept=True).fit(P) -design.transform(P.head()) -``` - -```{code-cell} ipython3 -for name, build in [('patsy', lambda: patsy.dmatrix('Price + US', P)), - ('formulaic', lambda: formulaic.model_matrix('Price + US', P))]: - try: - print(f'{name}: {type(build()).__name__}') - except Exception as e: - print(f'{name}: {type(e).__name__}') -``` - -## Model structure - -All three record which columns of the model matrix belong to each term, -which is what an ANOVA or a group-wise selection procedure needs. - -```{code-cell} ipython3 -design = MS(['Price', 'ShelveLoc', ('Price', 'ShelveLoc')], intercept=True).fit(Carseats) -print('ModelSpec', {str(k): (v.start, v.stop) for k, v in design.column_map_.items()}) -print('patsy ', {k: (v.start, v.stop) for k, v in - patsy.dmatrix('Price*ShelveLoc', Carseats).design_info.term_name_slices.items()}) -print('formulaic', {str(k): v for k, v in - formulaic.model_matrix('Price*ShelveLoc', Carseats).model_spec.term_indices.items()}) -``` - -Where they overlap, the matrices agree: treatment and sum contrasts, -and interactions such as `Price*ShelveLoc`, give the same columns up to -order. - -```{code-cell} ipython3 -def same_columns(A, B): - A, B = np.asarray(A, float), np.asarray(B, float) - return A.shape == B.shape and all(any(np.allclose(a, b) for b in B.T) for a in A.T) - -same_columns(MS(['Price', 'ShelveLoc', ('Price', 'ShelveLoc')], intercept=True).fit_transform(Carseats), - patsy.dmatrix('Price*ShelveLoc', Carseats)) -``` - -## Defaults that differ - -| | `ModelSpec` | `ColumnTransformer` | patsy | formulaic | -|---|---|---|---|---| -| a column of ones (intercept) | yes, with `intercept=True`, the default until ISLP 0.7 | no: `sklearn` estimators fit their own | yes | yes | -| missing values | propagate as `NaN` rows | passed through; `OneHotEncoder` codes a missing level as a level of its own | rows dropped | rows dropped | -| unused levels of a categorical | dropped | dropped (`OneHotEncoder` learns the levels it sees) | column of zeros | column of zeros | -| level unseen when fit | error | error (`OneHotEncoder`'s default) | error | encoded as all zeros, with a warning | -| string column, not categorical | error: use `astype('category')` or `contrast()` | as its transformer: `'passthrough'` keeps the strings | treated as categorical | treated as categorical | -| interactions without main effects | product of each term's coding | (no interactions) | recoded to keep the model full rank | recoded to keep the model full rank | - -`ModelSpec`'s column of ones is set to change: `intercept=True` is its -default only until ISLP 0.7, when `ModelSpec` moves to its own package -and the default becomes `intercept=False`, as for a `ColumnTransformer`. -Pass `intercept=True` for statsmodels' models (`OLS`, `GLM`, ...), which -need the column, and `intercept=False` for `sklearn` and glmstar -estimators, which fit their own intercept; the examples on this page -pass it explicitly for this reason. - -Ordered categoricals are coded with contrasts by `ModelSpec`, patsy and -formulaic. `ModelSpec` can -instead score one by the position of its level with `ordinal()`, which -requires an explicit order: an ordered categorical (or a polars `Enum`), -or `levels=[...]`. - -## Where the others go further - -- **Syntax.** A formula is shorter than the equivalent list of terms. - `ModelSpec` relies on Python to generate lists of terms instead. -- **R's coding rules.** Both recode the terms of a formula, e.g. an - interaction without its main effects, to keep the model matrix full - rank, as R does; `ModelSpec` codes each term as given. -- **Native backends.** formulaic materializes a model matrix from a - polars or pyarrow frame directly; `ModelSpec` converts the columns it - uses to `pandas` first, copying them. -- **Speed and memory.** For 2,000,000 rows, two numeric columns and a - categorical of 50 levels (a dense model matrix of 0.83 GB), formulaic - built the model matrix in 0.1 s with 0.9 GB of memory besides the data; - `ModelSpec` and a `ColumnTransformer` took about 0.5 s and 1.7 GB, - holding the largest block of columns and the model matrix at once. -- **No extra dependency.** A `ColumnTransformer` is part of `sklearn`, - with its own conventions for naming columns - (`get_feature_names_out`) and choosing them (`make_column_selector`). diff --git a/docs/source/models/spec.ipynb b/docs/source/models/spec.ipynb deleted file mode 100644 index 30228fd..0000000 --- a/docs/source/models/spec.ipynb +++ /dev/null @@ -1,1926 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "typical-correlation", - "metadata": {}, - "source": [ - "# Building design matrices with `ModelSpec`\n", - "\n", - "The `ISLP` package provides a facility to build design\n", - "matrices for regression and classification tasks. It provides similar functionality to the formula\n", - "notation of `R` though uses python objects rather than specification through the special formula syntax.\n", - "\n", - "Related tools include `patsy` and `ColumnTransformer` from `sklearn.compose`. \n", - "\n", - "Perhaps the most common use is to extract some columns from a `pd.DataFrame` and \n", - "produce a design matrix, optionally with an intercept." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "sticky-desperate", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec,\n", - " summarize,\n", - " Column,\n", - " Feature,\n", - " build_columns,\n", - " ordinal)\n", - "\n", - "import statsmodels.api as sm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "devoted-antique", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "b7a2e6ab-491d-4a57-8184-a9fcccb2047b", - "metadata": {}, - "source": [ - "We'll first build a design matrix that we can use to model `Sales`\n", - "in terms of the categorical variable `ShelveLoc` and `Price`.\n", - "\n", - "We see first that `ShelveLoc` is a categorical variable:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "7d3642a6-90c6-48ad-8d35-88231b4991f8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 Bad\n", - "1 Good\n", - "2 Medium\n", - "3 Medium\n", - "4 Bad\n", - " ... \n", - "395 Good\n", - "396 Medium\n", - "397 Medium\n", - "398 Bad\n", - "399 Good\n", - "Name: ShelveLoc, Length: 400, dtype: category\n", - "Categories (3, object): ['Bad', 'Good', 'Medium']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats['ShelveLoc']" - ] - }, - { - "cell_type": "markdown", - "id": "4afa201d-4b19-4d85-9e1b-1392a54d027b", - "metadata": {}, - "source": [ - "This is recognized by `ModelSpec` and only 2 columns are added for the three levels. The\n", - "default behavior is to drop the first level of the categories. Later, \n", - "we will show other contrasts of the 3 columns can be produced. \n", - "\n", - "This simple example below illustrates how the first argument (its `terms`) is\n", - "used to construct a design matrix." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "fd5528fe-11da-4e10-8996-06085896c1a0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptShelveLoc[Good]ShelveLoc[Medium]Price
01.00.00.0120
11.01.00.083
21.00.01.080
31.00.01.097
41.00.00.0128
51.00.00.072
61.00.01.0108
71.01.00.0120
81.00.01.0124
91.00.01.0124
\n", - "
" - ], - "text/plain": [ - " intercept ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 1.0 0.0 0.0 120\n", - "1 1.0 1.0 0.0 83\n", - "2 1.0 0.0 1.0 80\n", - "3 1.0 0.0 1.0 97\n", - "4 1.0 0.0 0.0 128\n", - "5 1.0 0.0 0.0 72\n", - "6 1.0 0.0 1.0 108\n", - "7 1.0 1.0 0.0 120\n", - "8 1.0 0.0 1.0 124\n", - "9 1.0 0.0 1.0 124" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True)\n", - "X = MS.fit_transform(Carseats)\n", - "X.iloc[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "6948e1ef-3685-4840-a4f2-ef15a1bcfb69", - "metadata": {}, - "source": [ - "We note that a column of ones, named `intercept`, has been added: `intercept=True`, as\n", - "statsmodels' models need it. Its default, `True` for now, will change to `False` in ISLP 0.6\n", - "(glmstar's estimators and most scikit-learn estimators fit the intercept themselves), so pass\n", - "`intercept` explicitly: leaving it out gives a `FutureWarning`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "682d4c81-eba9-467d-a176-911a0269a21d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Good]ShelveLoc[Medium]Price
00.00.0120
11.00.083
20.01.080
30.01.097
40.00.0128
50.00.072
60.01.0108
71.00.0120
80.01.0124
90.01.0124
\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 0.0 0.0 120\n", - "1 1.0 0.0 83\n", - "2 0.0 1.0 80\n", - "3 0.0 1.0 97\n", - "4 0.0 0.0 128\n", - "5 0.0 0.0 72\n", - "6 0.0 1.0 108\n", - "7 1.0 0.0 120\n", - "8 0.0 1.0 124\n", - "9 0.0 1.0 124" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS_no1 = ModelSpec(['ShelveLoc', 'Price'], intercept=False)\n", - "MS_no1.fit_transform(Carseats)[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "54d8fd20-d8f5-44d6-9965-83e745680798", - "metadata": {}, - "source": [ - "We see that `ShelveLoc` still only contributes\n", - "two columns to the design. The `ModelSpec` object does no introspection of its arguments to effectively include an intercept term\n", - "in the column space of the design matrix.\n", - "\n", - "To include this intercept via `ShelveLoc` we can use 3 columns to encode this categorical variable. Following the nomenclature of\n", - "`R`, we call this a `Contrast` of the categorical variable." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "555734bb-2682-4721-a1cd-6fb207394b0e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Bad]ShelveLoc[Good]ShelveLoc[Medium]Price
01.00.00.0120
10.01.00.083
20.00.01.080
30.00.01.097
41.00.00.0128
51.00.00.072
60.00.01.0108
70.01.00.0120
80.00.01.0124
90.00.01.0124
\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Bad] ShelveLoc[Good] ShelveLoc[Medium] Price\n", - "0 1.0 0.0 0.0 120\n", - "1 0.0 1.0 0.0 83\n", - "2 0.0 0.0 1.0 80\n", - "3 0.0 0.0 1.0 97\n", - "4 1.0 0.0 0.0 128\n", - "5 1.0 0.0 0.0 72\n", - "6 0.0 0.0 1.0 108\n", - "7 0.0 1.0 0.0 120\n", - "8 0.0 0.0 1.0 124\n", - "9 0.0 0.0 1.0 124" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from ISLP.models import contrast\n", - "shelve = contrast('ShelveLoc', None)\n", - "MS_contr = ModelSpec([shelve, 'Price'], intercept=False)\n", - "MS_contr.fit_transform(Carseats)[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "66db03cf-489c-40b6-8fac-762d66cf9932", - "metadata": {}, - "source": [ - "This example above illustrates that columns need not be identified by name in `terms`. The basic\n", - "role of an item in the `terms` sequence is a description of how to extract a column\n", - "from a columnar data object, usually a `pd.DataFrame`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "852ee40e-05d2-4785-ab7d-968fb087f3c0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Column(idx='ShelveLoc', name='ShelveLoc', is_categorical=True, is_ordinal=False, columns=(), encoder=Contrast(method=None))" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "shelve" - ] - }, - { - "cell_type": "markdown", - "id": "b3be8808-1dbf-4154-882b-f61656a2ed4e", - "metadata": {}, - "source": [ - "The `Column` object can be used to directly extract relevant columns from a `pd.DataFrame`. If the `encoder` field is not\n", - "`None`, then the extracted columns will be passed through `encoder`.\n", - "The `get_columns` method produces these columns as well as names for the columns.\n", - "The encoder of `shelve` itself has not been fit: fitting `MS_contr` fit a copy of it,\n", - "so that `shelve` can be reused in other specifications. We ask `get_columns` to fit it with `fit=True`." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "0ebadfc0-0ea2-4abc-aac6-ef78be227ce1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[1., 0., 0.],\n", - " [0., 1., 0.],\n", - " [0., 0., 1.],\n", - " ...,\n", - " [0., 0., 1.],\n", - " [1., 0., 0.],\n", - " [0., 1., 0.]], shape=(400, 3)),\n", - " ['ShelveLoc[Bad]', 'ShelveLoc[Good]', 'ShelveLoc[Medium]'])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "shelve.get_columns(Carseats, fit=True)" - ] - }, - { - "cell_type": "markdown", - "id": "269e6d18-4ae4-4a77-8498-90281ae7c803", - "metadata": {}, - "source": [ - "Let's now fit a simple OLS model with this design." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "411238d0-dd36-4878-a869-e8ce0ada099c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
coefstd errtP>|t|
ShelveLoc[Bad]12.00180.50323.8390.0
ShelveLoc[Good]16.89760.52232.3860.0
ShelveLoc[Medium]13.86380.48728.4670.0
Price-0.05670.004-13.9670.0
\n", - "
" - ], - "text/plain": [ - " coef std err t P>|t|\n", - "ShelveLoc[Bad] 12.0018 0.503 23.839 0.0\n", - "ShelveLoc[Good] 16.8976 0.522 32.386 0.0\n", - "ShelveLoc[Medium] 13.8638 0.487 28.467 0.0\n", - "Price -0.0567 0.004 -13.967 0.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X = MS_contr.transform(Carseats)\n", - "Y = Carseats['Sales']\n", - "M_ols = sm.OLS(Y, X).fit()\n", - "summarize(M_ols)" - ] - }, - { - "cell_type": "markdown", - "id": "40ddf68e-7d58-4e30-93a8-5b7fe840d37a", - "metadata": {}, - "source": [ - "## Interactions\n", - "\n", - "One of the common uses of formulae in `R` is to specify interactions between variables.\n", - "This is done in `ModelSpec` by including a tuple in the `terms` argument." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "3f5e314c-7a7f-4e8d-bb07-295beb42c728", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptShelveLoc[Bad]:PriceShelveLoc[Good]:PriceShelveLoc[Medium]:PricePrice
01.0120.00.00.0120
11.00.083.00.083
21.00.00.080.080
31.00.00.097.097
41.0128.00.00.0128
51.072.00.00.072
61.00.00.0108.0108
71.00.0120.00.0120
81.00.00.0124.0124
91.00.00.0124.0124
\n", - "
" - ], - "text/plain": [ - " intercept ShelveLoc[Bad]:Price ShelveLoc[Good]:Price \\\n", - "0 1.0 120.0 0.0 \n", - "1 1.0 0.0 83.0 \n", - "2 1.0 0.0 0.0 \n", - "3 1.0 0.0 0.0 \n", - "4 1.0 128.0 0.0 \n", - "5 1.0 72.0 0.0 \n", - "6 1.0 0.0 0.0 \n", - "7 1.0 0.0 120.0 \n", - "8 1.0 0.0 0.0 \n", - "9 1.0 0.0 0.0 \n", - "\n", - " ShelveLoc[Medium]:Price Price \n", - "0 0.0 120 \n", - "1 0.0 83 \n", - "2 80.0 80 \n", - "3 97.0 97 \n", - "4 0.0 128 \n", - "5 0.0 72 \n", - "6 108.0 108 \n", - "7 0.0 120 \n", - "8 124.0 124 \n", - "9 124.0 124 " - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ModelSpec([(shelve, 'Price'), 'Price'], intercept=True).fit_transform(Carseats).iloc[:10]" - ] - }, - { - "cell_type": "markdown", - "id": "3f85fcb2-f0ef-4c1b-a89f-fcf083937274", - "metadata": {}, - "source": [ - "The above design matrix is clearly rank deficient, as `ModelSpec` has not inspected the formula\n", - "and attempted to produce a corresponding matrix that may or may not match a user's intent." - ] - }, - { - "cell_type": "markdown", - "id": "excellent-hamilton", - "metadata": {}, - "source": [ - "## Ordinal variables\n", - "\n", - "Ordered categoricals, such as the income bands produced by `pd.cut()` below,\n", - "are encoded with contrasts like any other categorical, in the order of their\n", - "categories." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "going-administrator", - "metadata": {}, - "outputs": [], - "source": [ - "Carseats['OIncome'] = pd.cut(Carseats['Income'], \n", - " [0,50,90,200], \n", - " labels=['L','M','H'])\n", - "MS_order = ModelSpec(['OIncome'], intercept=True).fit(Carseats)" - ] - }, - { - "cell_type": "markdown", - "id": "5e1defb1-071b-4751-9358-b8d2f0b3412e", - "metadata": {}, - "source": [ - "Part of the `fit` method of `ModelSpec` involves inspection of the columns of `Carseats`. \n", - "The results of that inspection can be found in the `column_info_` attribute:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "050fb4ae-648d-429d-9cb2-8423ad9707d7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Sales': Column(idx='Sales', name='Sales', is_categorical=False, is_ordinal=False, columns=('Sales',), encoder=None),\n", - " 'CompPrice': Column(idx='CompPrice', name='CompPrice', is_categorical=False, is_ordinal=False, columns=('CompPrice',), encoder=None),\n", - " 'Income': Column(idx='Income', name='Income', is_categorical=False, is_ordinal=False, columns=('Income',), encoder=None),\n", - " 'Advertising': Column(idx='Advertising', name='Advertising', is_categorical=False, is_ordinal=False, columns=('Advertising',), encoder=None),\n", - " 'Population': Column(idx='Population', name='Population', is_categorical=False, is_ordinal=False, columns=('Population',), encoder=None),\n", - " 'Price': Column(idx='Price', name='Price', is_categorical=False, is_ordinal=False, columns=('Price',), encoder=None),\n", - " 'ShelveLoc': Column(idx='ShelveLoc', name='ShelveLoc', is_categorical=True, is_ordinal=False, columns=('ShelveLoc[Good]', 'ShelveLoc[Medium]'), encoder=Contrast()),\n", - " 'Age': Column(idx='Age', name='Age', is_categorical=False, is_ordinal=False, columns=('Age',), encoder=None),\n", - " 'Education': Column(idx='Education', name='Education', is_categorical=False, is_ordinal=False, columns=('Education',), encoder=None),\n", - " 'Urban': Column(idx='Urban', name='Urban', is_categorical=True, is_ordinal=False, columns=('Urban[Yes]',), encoder=Contrast()),\n", - " 'US': Column(idx='US', name='US', is_categorical=True, is_ordinal=False, columns=('US[Yes]',), encoder=Contrast()),\n", - " 'OIncome': Column(idx='OIncome', name='OIncome', is_categorical=True, is_ordinal=True, columns=('OIncome[M]', 'OIncome[H]'), encoder=Contrast())}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS_order.column_info_" - ] - }, - { - "cell_type": "markdown", - "id": "f2a6f087", - "metadata": {}, - "source": [ - "To instead encode an ordinal variable by a single column holding the\n", - "position of its level, use `ordinal()`. The order comes from an ordered\n", - "categorical, as here, or can be given with `levels=[...]`; for other\n", - "columns, such as strings, the order would be ambiguous and `ordinal()`\n", - "raises an error." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "0e5eb308", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
interceptordinal(OIncome)
01.01.0
11.00.0
21.00.0
31.02.0
41.01.0
\n", - "
" - ], - "text/plain": [ - " intercept ordinal(OIncome)\n", - "0 1.0 1.0\n", - "1 1.0 0.0\n", - "2 1.0 0.0\n", - "3 1.0 2.0\n", - "4 1.0 1.0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ModelSpec([ordinal('OIncome')], intercept=True).fit_transform(Carseats).iloc[:5]" - ] - }, - { - "cell_type": "markdown", - "id": "debf7e2e-0a9d-451b-866c-66c0df9f43e5", - "metadata": {}, - "source": [ - "## Structure of a `ModelSpec`\n", - "\n", - "The first argument to `ModelSpec` is stored as the `terms` attribute. Under the hood,\n", - "this sequence is inspected to produce the `terms_` attribute which specify the objects\n", - "that will ultimately create the design matrix." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "ea51e988-0857-4d49-9987-d7531b34a233", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Feature(variables=('ShelveLoc',), name='ShelveLoc', encoder=None, use_transform=True, pure_columns=True, override_encoder_colnames=False),\n", - " Feature(variables=('Price',), name='Price', encoder=None, use_transform=True, pure_columns=True, override_encoder_colnames=False)]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['ShelveLoc', 'Price'], intercept=True)\n", - "MS.fit(Carseats)\n", - "MS.terms_" - ] - }, - { - "cell_type": "markdown", - "id": "warming-mobile", - "metadata": {}, - "source": [ - "Each element of `terms_` should be a `Feature` which describes a set of columns to be extracted from\n", - "a columnar data form as well as possible a possible encoder." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "59214a70-1e6b-41c4-9f44-a92d340723c9", - "metadata": {}, - "outputs": [], - "source": [ - "shelve_var = MS.terms_[0]" - ] - }, - { - "cell_type": "markdown", - "id": "5fed3ea2-ff50-4e5d-819d-a948f121f9d3", - "metadata": {}, - "source": [ - "We can find the columns associated to each term using the `build_columns` method of `ModelSpec`:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "5e25ef64-497d-4f42-9f20-3d4a320cda23", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ShelveLoc[Good]ShelveLoc[Medium]
00.00.0
11.00.0
20.01.0
30.01.0
40.00.0
.........
3951.00.0
3960.01.0
3970.01.0
3980.00.0
3991.00.0
\n", - "

400 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " ShelveLoc[Good] ShelveLoc[Medium]\n", - "0 0.0 0.0\n", - "1 1.0 0.0\n", - "2 0.0 1.0\n", - "3 0.0 1.0\n", - "4 0.0 0.0\n", - ".. ... ...\n", - "395 1.0 0.0\n", - "396 0.0 1.0\n", - "397 0.0 1.0\n", - "398 0.0 0.0\n", - "399 1.0 0.0\n", - "\n", - "[400 rows x 2 columns]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df, names = build_columns(MS.column_info_,\n", - " Carseats, \n", - " shelve_var)\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "63edf7a2-e776-45b0-b434-d676d7e13dbd", - "metadata": {}, - "source": [ - "The design matrix is constructed by running through `terms_` and concatenating the corresponding columns." - ] - }, - { - "cell_type": "markdown", - "id": "former-spring", - "metadata": {}, - "source": [ - "### `Feature` objects\n", - "\n", - "Note that `Feature` objects have a tuple of `variables` as well as an `encoder` attribute. The\n", - "tuple of `variables` first creates a concatenated dataframe from all corresponding variables and then\n", - "is run through `encoder.transform`. The `encoder.fit` method of each `Feature` is run once during \n", - "the call to `ModelSpec.fit`." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "floral-liabilities", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
PriceIncomeOIncome
0120.073.02.0
183.048.01.0
280.035.01.0
397.0100.00.0
4128.064.02.0
............
395128.0108.00.0
396120.023.01.0
397159.026.01.0
39895.079.02.0
399120.037.01.0
\n", - "

400 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " Price Income OIncome\n", - "0 120.0 73.0 2.0\n", - "1 83.0 48.0 1.0\n", - "2 80.0 35.0 1.0\n", - "3 97.0 100.0 0.0\n", - "4 128.0 64.0 2.0\n", - ".. ... ... ...\n", - "395 128.0 108.0 0.0\n", - "396 120.0 23.0 1.0\n", - "397 159.0 26.0 1.0\n", - "398 95.0 79.0 2.0\n", - "399 120.0 37.0 1.0\n", - "\n", - "[400 rows x 3 columns]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=None)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " new_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "reasonable-canadian", - "metadata": {}, - "source": [ - "Let's now transform these columns with an encoder. Within `ModelSpec` we will first build the\n", - "arrays above and then call `pca.fit` and finally `pca.transform` within `design.build_columns`." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "imported-measure", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
mynewvar[0]mynewvar[1]
0-3.595740-4.850530
115.07040135.706773
227.41222840.772377
3-33.98304813.468087
46.580644-11.287452
.........
395-36.856308-18.418138
39645.7315203.243768
39749.087659-35.727136
398-13.56517818.847760
39931.9170720.976615
\n", - "

400 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " mynewvar[0] mynewvar[1]\n", - "0 -3.595740 -4.850530\n", - "1 15.070401 35.706773\n", - "2 27.412228 40.772377\n", - "3 -33.983048 13.468087\n", - "4 6.580644 -11.287452\n", - ".. ... ...\n", - "395 -36.856308 -18.418138\n", - "396 45.731520 3.243768\n", - "397 49.087659 -35.727136\n", - "398 -13.565178 18.847760\n", - "399 31.917072 0.976615\n", - "\n", - "[400 rows x 2 columns]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.decomposition import PCA\n", - "pca = PCA(n_components=2)\n", - "pca.fit(build_columns(MS.column_info_, Carseats, new_var)[0]) # this is done within `ModelSpec.fit`\n", - "pca_var = Feature(('Price', 'Income', 'OIncome'), name='mynewvar', encoder=pca)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " pca_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "institutional-burden", - "metadata": {}, - "source": [ - "The elements of the `variables` attribute may be column identifiers ( `\"Price\"`), `Column` instances (`price`)\n", - "or `Feature` instances (`pca_var`)." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "western-bloom", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IncomePricemynewvar[0]mynewvar[1]
073.0120.0-3.595740-4.850530
148.083.015.07040135.706773
235.080.027.41222840.772377
3100.097.0-33.98304813.468087
464.0128.06.580644-11.287452
...............
395108.0128.0-36.856308-18.418138
39623.0120.045.7315203.243768
39726.0159.049.087659-35.727136
39879.095.0-13.56517818.847760
39937.0120.031.9170720.976615
\n", - "

400 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " Income Price mynewvar[0] mynewvar[1]\n", - "0 73.0 120.0 -3.595740 -4.850530\n", - "1 48.0 83.0 15.070401 35.706773\n", - "2 35.0 80.0 27.412228 40.772377\n", - "3 100.0 97.0 -33.983048 13.468087\n", - "4 64.0 128.0 6.580644 -11.287452\n", - ".. ... ... ... ...\n", - "395 108.0 128.0 -36.856308 -18.418138\n", - "396 23.0 120.0 45.731520 3.243768\n", - "397 26.0 159.0 49.087659 -35.727136\n", - "398 79.0 95.0 -13.565178 18.847760\n", - "399 37.0 120.0 31.917072 0.976615\n", - "\n", - "[400 rows x 4 columns]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "price = MS.column_info_['Price']\n", - "fancy_var = Feature(('Income', price, pca_var), name='fancy', encoder=None)\n", - "build_columns(MS.column_info_,\n", - " Carseats, \n", - " fancy_var)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "e289feba-e3f5-48e0-9e29-cdd88d7f9923", - "metadata": {}, - "source": [ - "## Predicting at new points" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "6efed2fa-9e5d-429c-a8d9-ac544cab2b41", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "intercept 12.661546\n", - "Price -0.052213\n", - "Income 0.012829\n", - "dtype: float64" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MS = ModelSpec(['Price', 'Income'], intercept=True).fit(Carseats)\n", - "X = MS.transform(Carseats)\n", - "Y = Carseats['Sales']\n", - "M_ols = sm.OLS(Y, X).fit()\n", - "M_ols.params" - ] - }, - { - "cell_type": "markdown", - "id": "e6b4609b-fcb2-4cc2-b630-509df4c87546", - "metadata": {}, - "source": [ - "As `ModelSpec` is a transformer, it can be evaluated at new feature values.\n", - "Constructing the design matrix at any values is carried out by the `transform` method." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "8784b0e8-ce53-4a90-aee6-b935834295c7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([10.70130676, 10.307465 ])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_data = pd.DataFrame({'Price':[40, 50], 'Income':[10, 20]})\n", - "new_X = MS.transform(new_data)\n", - "M_ols.get_prediction(new_X).predicted_mean" - ] - }, - { - "cell_type": "markdown", - "id": "signal-yahoo", - "metadata": {}, - "source": [ - "## Using `np.ndarray`\n", - "\n", - "As the basic model is to concatenate columns extracted from a columnar data\n", - "representation, one *can* use `np.ndarray` as the column data. In this case,\n", - "columns will be selected by integer indices. \n", - "\n", - "### Caveats using `np.ndarray`\n", - "\n", - "If the `terms` only refer to a few columns of the data frame, the `transform` method only needs a dataframe with those columns.\n", - "However,\n", - "unless all features are floats, `np.ndarray` will default to a dtype of `object`, complicating issues.\n", - "\n", - "However, if we had used an `np.ndarray`, the column identifiers would be integers identifying specific columns so,\n", - "in order to work correctly, `transform` would need another `np.ndarray` where the columns have the same meaning. \n", - "\n", - "We illustrate this below, where we build a model from `Price` and `Income` for `Sales` and want to find predictions at new\n", - "values of `Price` and `Location`. We first find the predicitions using `pd.DataFrame` and then illustrate the difficulties\n", - "in using `np.ndarray`." - ] - }, - { - "cell_type": "markdown", - "id": "e7ffdd07-4d6b-4a4c-ab38-ad1270e85de6", - "metadata": {}, - "source": [ - "We will refit this model, using `ModelSpec` with an `np.ndarray` instead" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "4fec9030-7445-48be-a15f-2ac5a789e717", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 1., 120., 73.],\n", - " [ 1., 83., 48.],\n", - " [ 1., 80., 35.],\n", - " ...,\n", - " [ 1., 159., 26.],\n", - " [ 1., 95., 79.],\n", - " [ 1., 120., 37.]])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats_np = np.asarray(Carseats[['Price', 'Education', 'Income']])\n", - "MS_np = ModelSpec([0,2], intercept=True).fit(Carseats_np)\n", - "MS_np.transform(Carseats_np)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "c864e365-2476-4ca6-9d27-625cac2b2271", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "const 12.661546\n", - "x1 -0.052213\n", - "x2 0.012829\n", - "dtype: float64" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "M_ols_np = sm.OLS(Y, MS_np.transform(Carseats_np)).fit()\n", - "M_ols_np.params" - ] - }, - { - "cell_type": "markdown", - "id": "undefined-sacrifice", - "metadata": {}, - "source": [ - "Now, let's consider finding the design matrix at new points. \n", - "When using `pd.DataFrame` we only need to supply the `transform` method\n", - "a data frame with columns implicated in the `terms` argument (in this case, `Price` and `Income`). \n", - "\n", - "However, when using `np.ndarray` with integers as indices, `Price` was column 0 and `Income` was column 2. The only\n", - "sensible way to produce a return for predict is to extract its 0th and 2nd columns. Note this means\n", - "that the meaning of columns in an `np.ndarray` provided to `transform` essentially must be identical to those\n", - "passed to `fit`. To guard against this, `transform` requires an `np.ndarray` with as many columns as it was fit with." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "incredible-concert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "X has 2 features, but ModelSpec is expecting 3 features as input.\n" - ] - } - ], - "source": [ - "try:\n", - " new_D = np.array([[40,50], [10,20]]).T\n", - " new_X = MS_np.transform(new_D)\n", - "except ValueError as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "id": "allied-botswana", - "metadata": {}, - "source": [ - "Ultimately, `M` expects 3 columns for new predictions because it was fit\n", - "with a matrix having 3 columns (the first representing an intercept).\n", - "\n", - "We might be tempted to try as with the `pd.DataFrame` and produce\n", - "an `np.ndarray` with only the necessary variables." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "stunning-container", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 1. 40. 10.]\n", - " [ 1. 50. 20.]]\n" - ] - }, - { - "data": { - "text/plain": [ - "array([10.70130676, 10.307465 ])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_D = np.array([[40,50], [np.nan, np.nan], [10,20]]).T\n", - "new_X = MS_np.transform(new_D)\n", - "print(new_X)\n", - "M_ols.get_prediction(new_X).predicted_mean" - ] - }, - { - "cell_type": "markdown", - "id": "specific-tobacco", - "metadata": {}, - "source": [ - "For more complicated design contructions ensuring the columns of `new_D` match that of the original data will be more cumbersome. We expect\n", - "then that `pd.DataFrame` (or a columnar data representation with similar API) will likely be easier to use with `ModelSpec`." - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/models///ipynb,jupyterbook/models///md:myst,jupyterbook/models///ipynb" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/transforms.rst b/docs/source/transforms.rst index 8e18288..f923d07 100644 --- a/docs/source/transforms.rst +++ b/docs/source/transforms.rst @@ -1,8 +1,11 @@ Transforms for flexible features ================================ +The transforms are those of `modelmatrix `_, whose +documentation has these pages. + .. toctree:: - transforms/poly - transforms/splines - transforms/PCA + Polynomials + Splines + Principal components diff --git a/docs/source/transforms/PCA.ipynb b/docs/source/transforms/PCA.ipynb deleted file mode 100644 index a71eafe..0000000 --- a/docs/source/transforms/PCA.ipynb +++ /dev/null @@ -1,244 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "25478d81", - "metadata": {}, - "source": [ - "# Derived features: using PCA on a subset of columns\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of transformers applied to features." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "6d8987d9", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from ISLP import load_data\n", - "from ISLP.models import (ModelSpec, \n", - " pca, \n", - " Feature, \n", - " derived_feature,\n", - " build_columns)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "6674de9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "da655f52-6f3a-4762-95ed-9b02e158bf16", - "metadata": {}, - "source": [ - "Let's create a `ModelSpec` that is aware of all of the relevant columns." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "c9afb776-ae2b-48e0-9df8-e35dc7a81327", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec(Carseats.columns.drop(['Sales']), intercept=True).fit(Carseats)" - ] - }, - { - "cell_type": "markdown", - "id": "fff603bf", - "metadata": {}, - "source": [ - "Suppose we want to make a `Feature` representing the first 3 principal components of the\n", - " features `['CompPrice', 'Income', 'Advertising', 'Population', 'Price']`." - ] - }, - { - "cell_type": "markdown", - "id": "eab49ad1-3957-478f-8a76-28a8f58551e9", - "metadata": {}, - "source": [ - "We first make a `Feature` that represents these five features columns, then `pca`\n", - "can be used to compute a new `Feature` that returns the first three principal components." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "3e6901e9-470d-41a3-8a7d-64a8da7d80fa", - "metadata": {}, - "outputs": [], - "source": [ - "grouped = Feature(('CompPrice', 'Income', 'Advertising', 'Population', 'Price'), name='grouped', encoder=None)\n", - "sklearn_pca = PCA(n_components=3, whiten=True)" - ] - }, - { - "cell_type": "markdown", - "id": "b45655a3-393d-4b4c-b754-cda61ed0e014", - "metadata": {}, - "source": [ - "We can now fit `sklearn_pca` and create our new feature." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "6cfe8861-ad07-47b9-95d1-5d5513ff6fbe", - "metadata": {}, - "outputs": [], - "source": [ - "grouped_features = build_columns(design.column_info_,\n", - " Carseats,\n", - " grouped)[0]\n", - "sklearn_pca.fit(grouped_features) \n", - "pca_var = derived_feature(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'],\n", - " name='pca(grouped)', encoder=sklearn_pca)\n", - "derived_features, _ = build_columns(design.column_info_,\n", - " Carseats, \n", - " pca_var,\n", - " encoders=design.encoders_)" - ] - }, - { - "cell_type": "markdown", - "id": "1a882cbc-8e90-4779-b14f-6fd49f5651df", - "metadata": {}, - "source": [ - "## Helper function\n", - "\n", - "The function `pca` encompasses these steps into a single function for convenience." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9f4b0955", - "metadata": {}, - "outputs": [], - "source": [ - "group_pca = pca(['CompPrice', 'Income', 'Advertising', 'Population', 'Price'], \n", - " n_components=3, \n", - " whiten=True, \n", - " name='grouped')" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "6b382699-eb86-457f-8e91-09a63eb21d49", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['pca(grouped, n_components=3, whiten=True)[0]',\n", - " 'pca(grouped, n_components=3, whiten=True)[1]',\n", - " 'pca(grouped, n_components=3, whiten=True)[2]'],\n", - " dtype='object')" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pca_design = ModelSpec([group_pca], intercept=False)\n", - "ISLP_features = pca_design.fit_transform(Carseats)\n", - "ISLP_features.columns" - ] - }, - { - "cell_type": "markdown", - "id": "598e278c", - "metadata": {}, - "source": [ - "## Direct comparison" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "4a8d9b28", - "metadata": {}, - "outputs": [], - "source": [ - "X = np.asarray(Carseats[['CompPrice', 'Income', 'Advertising', 'Population', 'Price']])\n", - "sklearn_features = sklearn_pca.fit_transform(X)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "6efa6c67-86e1-4f51-86c2-25c838a90bf4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(4.073428490498941e-14, 0.0)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(ISLP_features - sklearn_features), np.linalg.norm(ISLP_features - np.asarray(derived_features))" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/transforms/poly.ipynb b/docs/source/transforms/poly.ipynb deleted file mode 100644 index 45c862e..0000000 --- a/docs/source/transforms/poly.ipynb +++ /dev/null @@ -1,341 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "69c2a17b", - "metadata": {}, - "source": [ - "# Polynomial features\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of orthogonal polynomials of features.\n", - "\n", - "Force rebuild" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "4651f149", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from ISLP import load_data\n", - "from ISLP.models import ModelSpec, poly" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "290af7e0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "1b68208a-218c-4ee9-a7d2-8a0b0451b03b", - "metadata": {}, - "source": [ - "Let's make a term representing a quartic effect for `Population`." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8c85fa88-cd27-4a83-9060-dd8dc178c83f", - "metadata": {}, - "outputs": [], - "source": [ - "quartic = poly('Population', 4)" - ] - }, - { - "cell_type": "markdown", - "id": "8e1df9d6-b2d3-42aa-addf-1155b30973e5", - "metadata": {}, - "source": [ - "The object `quartic` does not refer to any data yet, it must be included in a `ModelSpec` object\n", - "and fit using the `fit` method." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a49ad3e4-af07-4952-9a4a-590fca565d2c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['poly(Population, degree=4)[0]', 'poly(Population, degree=4)[1]',\n", - " 'poly(Population, degree=4)[2]', 'poly(Population, degree=4)[3]'],\n", - " dtype='object')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "design = ModelSpec([quartic], intercept=False)\n", - "ISLP_features = design.fit_transform(Carseats)\n", - "ISLP_features.columns" - ] - }, - { - "cell_type": "markdown", - "id": "9df16fd2-6427-4f1e-97d8-1d3e28ff1f0f", - "metadata": {}, - "source": [ - "## Compare to `R`\n", - "\n", - "We can compare our polynomials to a similar function in `R`" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "d6fde88f-cde3-43f1-8814-36ce4a4b4ecc", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext rpy2.ipython" - ] - }, - { - "cell_type": "markdown", - "id": "a62ef797-8c9c-45e4-a08d-b2efa8df709e", - "metadata": {}, - "source": [ - "We'll recompute these features using `poly` in `R`." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "1bc583ca-6325-4419-8ef5-cc42f970b67a", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats -o R_features\n", - "R_features = poly(Carseats$Population, 4)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "052f2e9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3.95855733196009e-15" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(ISLP_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "c4e1679d-6434-45e3-a675-33c1ad183f04", - "metadata": {}, - "source": [ - "## Underlying model\n", - "\n", - "If we look at `quartic`, we see it is a `Feature`, i.e. it can be used to produce a set of columns\n", - "in a design matrix when it is a term used in creating the `ModelSpec`.\n", - "\n", - "Its encoder is `Poly(degree=4)`. This is a special `sklearn` transform that expects a single column\n", - "in its `fit()` method and constructs a matrix of corresponding orthogonal polynomials.\n", - "\n", - "The spline helpers `ns` and `bs` as well as `pca` follow a similar structure." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "864c8a30-167c-40f8-a5b1-b26ba158aa21", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable(variables=('Population',), name='poly(Population, degree=4)', encoder=Poly(degree=4), use_transform=True, pure_columns=False, override_encoder_colnames=True)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "quartic" - ] - }, - { - "cell_type": "markdown", - "id": "9c31a4e6-11f2-4d90-9112-25756f88cf24", - "metadata": {}, - "source": [ - "## Raw polynomials\n", - "\n", - "One can compute raw polynomials (which results in a less well-conditioned design matrix) of course." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "c348c5b6-d542-48bc-8da0-cdebf404d331", - "metadata": {}, - "outputs": [], - "source": [ - "quartic_raw = poly('Population', degree=4, raw=True)" - ] - }, - { - "cell_type": "markdown", - "id": "8eea8b00-ed2b-4cdf-90dc-bf1d7406733d", - "metadata": {}, - "source": [ - "Let's compare the features again." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "07053253-7d7b-474a-a319-7982e052fb75", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([quartic_raw], intercept=False)\n", - "raw_features = design.fit_transform(Carseats)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "986583bb-b819-4d1e-bc4a-1288f94c16ab", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats -o R_features\n", - "R_features = poly(Carseats$Population, 4, raw=TRUE)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "00abce93-42ef-4f28-b9f7-8e1c7e24b3a5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(raw_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "2097fc9b-62ae-4ee7-aea4-0d65f9a07db3", - "metadata": {}, - "source": [ - "## Intercept\n", - "\n", - "Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, \n", - "it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True`" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "a2d6b36c-29ff-4d5e-ae1e-a7cab4bc68b4", - "metadata": {}, - "outputs": [], - "source": [ - "quartic_int = poly('Population', degree=4, raw=True, intercept=True)\n", - "design = ModelSpec([quartic_int], intercept=False)\n", - "intercept_features = design.fit_transform(Carseats)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "1ef711fd-5be2-4267-9a79-1553e4da91e9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(intercept_features.iloc[:,1:] - R_features)" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/transforms/splines.ipynb b/docs/source/transforms/splines.ipynb deleted file mode 100644 index 399b0be..0000000 --- a/docs/source/transforms/splines.ipynb +++ /dev/null @@ -1,332 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4a50ef04", - "metadata": {}, - "source": [ - "# Spline features\n", - "\n", - "The modelling tools included in `ISLP` allow for\n", - "construction of spline functions of features.\n", - "\n", - "Force rebuild" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "666e6a04", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from ISLP import load_data\n", - "from ISLP.models import ModelSpec, ns, bs" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "0ecda36e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price',\n", - " 'ShelveLoc', 'Age', 'Education', 'Urban', 'US'],\n", - " dtype='object')" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Carseats = load_data('Carseats')\n", - "Carseats.columns" - ] - }, - { - "cell_type": "markdown", - "id": "20c6e551", - "metadata": {}, - "source": [ - "Let's make a term representing a cubic spline for `Population`. We'll use knots based on the \n", - "deciles." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "31f3f4ff-a63f-4374-b484-fbd3b27ce1ad", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 58.9, 110.4, 160. , 218.6, 272. , 317.8, 366. , 412.2, 467. ])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "knots = np.percentile(Carseats['Population'], np.linspace(10, 90, 9))\n", - "knots" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1ea79e0d", - "metadata": {}, - "outputs": [], - "source": [ - "bs_pop = bs('Population', internal_knots=knots, degree=3)" - ] - }, - { - "cell_type": "markdown", - "id": "a47d2caa", - "metadata": {}, - "source": [ - "The object `bs_pop` does not refer to any data yet, it must be included in a `ModelSpec` object\n", - "and fit using the `fit` method." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "90e8f810", - "metadata": {}, - "outputs": [], - "source": [ - "design = ModelSpec([bs_pop], intercept=False)\n", - "py_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "markdown", - "id": "f98f658a", - "metadata": {}, - "source": [ - "## Compare to `R`\n", - "\n", - "We can compare our polynomials to a similar function in `R`" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "fda53527", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext rpy2.ipython" - ] - }, - { - "cell_type": "markdown", - "id": "20c76684", - "metadata": {}, - "source": [ - "We'll recompute these features using `bs` in `R`. The default knot selection of the\n", - "`ISLP` and `R` version are slightly different so we just fix the set of internal knots." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "5d5e7985", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -i Carseats,knots -o R_features\n", - "library(splines)\n", - "R_features = bs(Carseats$Population, knots=knots, degree=3)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "ceffa80f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1.1372379284497324e-15" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(py_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "a89ec276-cbf5-4fed-9720-c33060570fd9", - "metadata": {}, - "source": [ - "## Underlying model\n", - "\n", - "As for `poly`, the computation of the B-splines is done by a special `sklearn` transformer." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "cacf2134-70ad-48d7-aa1c-83c2c88d05f9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable(variables=('Population',), name='bs(Population, internal_knots=[ 58.9 110.4 160. 218.6 272. 317.8 366. 412.2 467. ], degree=3)', encoder=BSpline(internal_knots=array([ 58.9, 110.4, 160. , 218.6, 272. , 317.8, 366. , 412.2, 467. ]),\n", - " lower_bound=10.0, upper_bound=509.0), use_transform=True, pure_columns=False, override_encoder_colnames=True)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bs_pop" - ] - }, - { - "cell_type": "markdown", - "id": "7bd02961-3f17-4352-af69-c8229656ae96", - "metadata": {}, - "source": [ - "## Natural splines \n", - "\n", - "Natural cubic splines are also implemented." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "3c7c2445", - "metadata": {}, - "outputs": [], - "source": [ - "ns_pop = ns('Population', internal_knots=knots)\n", - "design = ModelSpec([ns_pop], intercept=False)\n", - "py_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "805060ca-5aac-46b4-842e-bbf4f39dc429", - "metadata": {}, - "outputs": [], - "source": [ - "%%R -o R_features\n", - "library(splines)\n", - "R_features = ns(Carseats$Population, knots=knots)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "6fb40058-6a80-431a-8c45-448d742d5947", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1.2473757226554746e-15" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.linalg.norm(py_features - R_features)" - ] - }, - { - "cell_type": "markdown", - "id": "39bc9180", - "metadata": {}, - "source": [ - "## Intercept\n", - "\n", - "Looking at `py_features` we see it contains columns: `[Population**i for i in range(1, 4)]`. That is, \n", - "it doesn't contain an intercept, the order 0 term. This can be include with `intercept=True`. This means that the\n", - "column space includes an intercept, though there is no specific column labeled as intercept." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "38de6cf7", - "metadata": {}, - "outputs": [], - "source": [ - "bs_int = ns('Population', internal_knots=knots, intercept=True)\n", - "design = ModelSpec([bs_int], intercept=False)\n", - "py_int_features = np.asarray(design.fit_transform(Carseats))" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "1dcd7ff8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((400, 11), (400, 10))" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "py_int_features.shape, py_features.shape" - ] - } - ], - "metadata": { - "jupytext": { - "formats": "source/transforms///ipynb,jupyterbook/transforms///md:myst,jupyterbook/transforms///ipynb" - }, - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pyproject.toml b/pyproject.toml index e9b59f9..0aed992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "ISLP" dependencies = ["numpy>=1.7.1", "scipy>=0.9", "pandas>=1.5", - "narwhals>=1.26", # dataframes other than pandas in ModelSpec + "modelmatrix>=0.3", # ModelSpec, moved to its own package "lxml", # pandas needs this for html "scikit-learn>=1.2", "joblib", @@ -87,8 +87,6 @@ build-backend = "setuptools.build_meta" [dependency-groups] dev = [ "flake8>=7.3.0", - "polars>=1.0", # ModelSpec with other dataframe backends - "pyarrow>=14", "pytest>=9.0.2", "pytest-cov>=6.0", "torchinfo>=1.8.0", @@ -99,8 +97,3 @@ dev = [ markers = [ "network: downloads data (run in CI only on the weekly schedule; deselect with -m 'not network')", ] -# ModelSpec warns at fit that it moves to its own package in ISLP 0.7; -# tests/models/test_intercept_default.py checks the warning itself -filterwarnings = [ - "ignore:ModelSpec will move:FutureWarning", -] diff --git a/tests/models/test_boolean_columns.py b/tests/models/test_boolean_columns.py deleted file mode 100644 index 48821ee..0000000 --- a/tests/models/test_boolean_columns.py +++ /dev/null @@ -1,27 +0,0 @@ -import pandas as pd -import statsmodels.api as sm -import numpy as np -from itertools import combinations - -from ISLP.models import ModelSpec as MS - -rng = np.random.default_rng(0) - -df = pd.DataFrame({'A':rng.standard_normal(10), - 'B':np.array([1,2,3,2,1,1,1,3,2,1], int), - 'C':np.array([True,False,False,True,True]*2, bool), - 'D':rng.standard_normal(10)}) -Y = rng.standard_normal(10) - -def test_all(): - - for i in range(1, 5): - for comb in combinations(['A','B','C','D'], i): - - X = MS(comb, intercept=True).fit_transform(df) - # issue #17: boolean columns should give a numeric design - assert list(X.columns) == ['intercept'] + list(comb) - assert all(np.issubdtype(t, np.number) for t in X.dtypes) - results = sm.OLS(Y, X).fit() - assert np.all(np.isfinite(results.params)) - diff --git a/tests/models/test_columns.py b/tests/models/test_columns.py deleted file mode 100644 index aa5aa32..0000000 --- a/tests/models/test_columns.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np, pandas as pd -from sklearn.base import clone - -from pandas.api.types import CategoricalDtype -from ISLP.models.columns import _get_column_info -from ISLP.models.model_spec import Contrast - -def test_column_info(): - - rng = np.random.default_rng(0) - cat_type = CategoricalDtype(categories=list("abcd"), ordered=True) - df = pd.DataFrame(rng.standard_normal((50, 5)), columns=['Aa', 'B', 'Ccc', 'D', 'E']) - df['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - df['D'] = pd.Categorical(rng.choice(['a','b','c','d'], 50, replace=True)) - df['D'].astype(cat_type) - column_info = _get_column_info(df, - df.columns, - [False]*4+[True], - [False]*5, - categorical_encoders={'categorical':Contrast(method='drop')}) - assert list(column_info) == ['Aa', 'B', 'Ccc', 'D', 'E'] - for col in ['Aa', 'B', 'Ccc', 'D']: - assert not column_info[col].is_categorical - assert column_info[col].columns == (col,) - # 'drop' removes the first level of E - assert column_info['E'].is_categorical - assert column_info['E'].columns == ('E[5]', 'E[6]', 'E[7]') - - diff --git a/tests/models/test_intercept_default.py b/tests/models/test_intercept_default.py deleted file mode 100644 index a24034f..0000000 --- a/tests/models/test_intercept_default.py +++ /dev/null @@ -1,113 +0,0 @@ -# The default `intercept=None` of ModelSpec: deprecated, it behaves as -# True but warns at fit, as the default will change to False in ISLP 0.7, -# when ModelSpec moves to its own package. Every fit warns of the move; -# only a spec relying on the default gets the advice on `intercept`. - -import warnings - -import numpy as np -import pytest -from sklearn.base import clone -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import cross_val_score -from sklearn.pipeline import make_pipeline - -from ISLP import load_data -from ISLP.models import ModelSpec as MS - -ADVICE = 'relies on the default of `intercept`' - - -def _future_warnings(record): - return [str(w.message) for w in record if issubclass(w.category, FutureWarning)] - - -@pytest.fixture(scope='module') -def Carseats(): - return load_data('Carseats') - - -TERMS = ['Price', 'ShelveLoc'] - - -def test_default_warns_once_per_fit(Carseats): - spec = MS(TERMS) - assert spec.intercept is None - with pytest.warns(FutureWarning, match='default to False') as record: - spec.fit(Carseats) - # one warning: of the move, with the advice on intercept - messages = _future_warnings(record) - assert len(messages) == 1 - assert messages[0].startswith('ModelSpec will move') and ADVICE in messages[0] - # transform and the other methods do not warn again - with warnings.catch_warnings(): - warnings.simplefilter('error', FutureWarning) - X = spec.transform(Carseats) - names = spec.names - list(spec.build_sequence(Carseats)) - spec.build_submodel(Carseats, spec.terms_[:1]) - assert spec.intercept is None - - -def test_default_behaves_as_True(Carseats): - with pytest.warns(FutureWarning): - spec = MS(TERMS).fit(Carseats) - X = spec.transform(Carseats) - X_true = MS(TERMS, intercept=True).fit_transform(Carseats) - assert list(X.columns) == list(X_true.columns) - assert X.columns[0] == 'intercept' - np.testing.assert_allclose(X['intercept'], 1) - np.testing.assert_allclose(X, X_true) - assert spec.names == ['intercept'] + TERMS - assert spec.column_map_['intercept'] == slice(0, 1) - D = spec.build_submodel(Carseats, spec.terms_[:1]) - assert list(D.columns) == ['intercept', 'Price'] - seq = list(spec.build_sequence(Carseats)) - assert list(seq[0].columns) == ['intercept'] - - -@pytest.mark.parametrize('intercept', [True, False]) -def test_explicit_warns_only_of_move(Carseats, intercept): - spec = MS(TERMS, intercept=intercept) - with pytest.warns(FutureWarning) as record: - spec.fit(Carseats) - messages = _future_warnings(record) - assert len(messages) == 1 - assert messages[0].startswith('ModelSpec will move') and ADVICE not in messages[0] - assert "filterwarnings('ignore', message='ModelSpec will move')" in messages[0] - with warnings.catch_warnings(): - warnings.simplefilter('error', FutureWarning) - X = spec.transform(Carseats) - spec.names - list(spec.build_sequence(Carseats)) - assert ('intercept' in X.columns) == intercept - assert ('intercept' in spec.names) == intercept - - -@pytest.mark.parametrize('intercept', [True, False]) -def test_clone_cross_val_score_explicit_no_advice(Carseats, intercept): - # the move warning can be silenced; nothing else then warns - spec = MS(TERMS, intercept=intercept) - pipe = make_pipeline(spec, LinearRegression(fit_intercept=not intercept)) - with warnings.catch_warnings(): - warnings.simplefilter('error', FutureWarning) - warnings.filterwarnings('ignore', message='ModelSpec will move') - clone(spec).fit(Carseats) - scores = cross_val_score(pipe, Carseats, Carseats['Sales'], cv=3, - error_score='raise') - assert np.all(np.isfinite(scores)) - - -def test_clone_get_params_preserve_None(Carseats): - spec = MS(TERMS) - assert spec.get_params()['intercept'] is None - with warnings.catch_warnings(): - # clone re-runs __init__: no warning until fit - warnings.simplefilter('error', FutureWarning) - cloned = clone(spec) - assert cloned.get_params()['intercept'] is None - with pytest.warns(FutureWarning): - spec.fit(Carseats) - # fitting does not resolve the parameter in place - assert spec.get_params()['intercept'] is None - assert clone(spec).get_params()['intercept'] is None diff --git a/tests/models/test_model_matrix.py b/tests/models/test_model_matrix.py deleted file mode 100644 index d38c16f..0000000 --- a/tests/models/test_model_matrix.py +++ /dev/null @@ -1,370 +0,0 @@ -import numpy as np, pandas as pd -from sklearn.base import clone - -from ISLP.transforms import Poly, NaturalSpline, BSpline, Interaction -from ISLP.models.model_spec import ModelSpec, Feature, ns, bs, poly, pca, contrast, Contrast, build_model - -from sklearn.preprocessing import (OneHotEncoder, - OrdinalEncoder) -from sklearn.decomposition import PCA - -default_encoders = {'categorical': Contrast(method=None), - 'ordinal': OrdinalEncoder()} - - -def test_interaction(): - - rng = np.random.default_rng(0) - I = Interaction(['V', 'U'], - {'V':[0,2], - 'U':[1,3,5]}, - {'V':[0,1],'U':[0,1,2]}) - X = rng.standard_normal((50,10)) - W = I.fit_transform(X) - - W2 = np.array([X[:,0]*X[:,1], - X[:,0]*X[:,3], - X[:,0]*X[:,5], - X[:,2]*X[:,1], - X[:,2]*X[:,3], - X[:,2]*X[:,5]]).T - np.testing.assert_allclose(W, W2) - assert list(W.columns) == ['0:0', '0:1', '0:2', '1:0', '1:1', '1:2'] - -def test_ndarray(): - - rng = np.random.default_rng(1) - X = rng.standard_normal((50,5)) - - M = ModelSpec(terms=[1, (3,2)], - categorical_encoders=default_encoders, intercept=True) - M.fit(X) - MX = M.transform(X) - - np.testing.assert_allclose(X[:,1], MX[:,1]) - np.testing.assert_allclose(X[:,2] * X[:,3], MX[:,2]) - -def test_dataframe1(): - - rng = np.random.default_rng(2) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - - M = ModelSpec(terms=['A','D',('D','E')], - categorical_encoders=default_encoders, intercept=True) - clone(M) - MX = np.asarray(M.fit_transform(D)) - - np.testing.assert_allclose(X[:,0], MX[:,1]) - np.testing.assert_allclose(X[:,3], MX[:,2]) - np.testing.assert_allclose(X[:,3]*X[:,4], MX[:,3]) - -def test_dataframe2(): - - rng = np.random.default_rng(3) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['V','B','A','D','E']) - - M = ModelSpec(terms=['A', 'D', 'B', ('D','E'), 'V'], - categorical_encoders=default_encoders, intercept=True) - clone(M) - - MX = M.fit_transform(D) - - X2 = D.copy() - MX2 = M.transform(D) - np.testing.assert_allclose(MX, MX2) - -def test_dataframe3(): - - rng = np.random.default_rng(8) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=['A', 'E', ('D','E')], - categorical_encoders=default_encoders, intercept=True) - MX = np.asarray(M.fit_transform(D)) - M2 = clone(M) - - DE = pd.get_dummies(D['E']) - np.testing.assert_allclose(X[:,0], MX[:,1]) - np.testing.assert_allclose(DE, MX[:,2:6]) - np.testing.assert_allclose(np.asarray(DE) * np.multiply.outer(X[:,3], np.ones(4)), MX[:,-4:]) - - X2 = D.copy() - MX2 = M.transform(D) - np.testing.assert_allclose(MX, MX2) - -def test_dataframe4(): - - rng = np.random.default_rng(9) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=['A', 'E', ('D','E'), 'D'], - categorical_encoders=default_encoders, intercept=True) - MX = np.asarray(M.fit_transform(D)) - - DE = pd.get_dummies(D['E']) - np.testing.assert_allclose(X[:,0], MX[:,1]) - np.testing.assert_allclose(DE, MX[:,2:6]) - - # check they agree on copy of dataframe - - X2 = D.copy() - MX2 = M.transform(D) - np.testing.assert_allclose(MX, MX2) - - print(MX2.columns) - -def test_dataframe5(): - - rng = np.random.default_rng(10) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=['A', 'E', ('D','E')], - categorical_encoders=default_encoders, intercept=True) - MX = np.asarray(M.fit_transform(D)) - - # check they agree on copy of dataframe - - X2 = D.copy() - MX2 = M.transform(D) - np.testing.assert_allclose(MX, MX2) - -def test_dataframe6(): - - rng = np.random.default_rng(11) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - W = Feature(('A','E'), 'AE', None) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=['A',W,(W,'D',)], - categorical_encoders=default_encoders, intercept=True) - MX = M.fit_transform(D) - - # intercept, A, AE (A and 4 levels of E), AE:D (5 x 3 levels of D) - assert MX.shape == (50, 1 + 1 + 5 + 15) - MX = np.asarray(MX) - DE = np.asarray(pd.get_dummies(D['E']), float) - DD = np.asarray(pd.get_dummies(D['D']), float) - np.testing.assert_allclose(MX[:,M.column_map_[W]], - np.column_stack([D['A'], DE])) - AE = np.column_stack([D['A'], DE]) - np.testing.assert_allclose(MX[:,M.column_map_[(W,'D')]], - np.einsum('ij,ik->ijk', AE, DD).reshape((50, -1))) - -def test_dataframe7(): - - rng = np.random.default_rng(12) - X = rng.standard_normal((50,6)) - D = pd.DataFrame(X, columns=['AA','Bbbb','C','Ddd','Y','Eee']) - D['Ddd'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['Eee'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=D.columns.drop(['Y','C']), - categorical_encoders=default_encoders, intercept=True) - MX = M.fit_transform(D) - assert list(MX.columns) == ['intercept', 'AA', 'Bbbb', - 'Ddd[a]', 'Ddd[b]', 'Ddd[c]', - 'Eee[4]', 'Eee[5]', 'Eee[6]', 'Eee[7]'] - np.testing.assert_allclose(MX[['AA', 'Bbbb']], D[['AA', 'Bbbb']]) - -def test_dataframe8(): - - rng = np.random.default_rng(13) - X = rng.standard_normal((50,6)) - D = pd.DataFrame(X, columns=['A','B','C','D','Y','E']) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - poly = Poly(degree=3) - W = Feature(('A',), 'poly(A)', poly) - M = ModelSpec(terms=list(D.columns.drop(['Y','C'])) + [(W,'E')], - categorical_encoders=default_encoders, intercept=True) - MX = M.fit_transform(D) - - # main effects: intercept, A, B, 3 levels of D, 4 levels of E; - # then 3 poly columns x 4 levels of E - assert MX.shape == (50, 1 + 2 + 3 + 4 + 12) - P = np.asarray(Poly(degree=3).fit_transform(D[['A']])) - DE = np.asarray(pd.get_dummies(D['E']), float) - np.testing.assert_allclose(np.asarray(MX)[:,-12:], - np.einsum('ij,ik->ijk', P, DE).reshape((50, -1))) - -def test_dataframe9(): - - rng = np.random.default_rng(14) - X = rng.standard_normal((50,6)) - D = pd.DataFrame(X, columns=['A','B','C','D','Y','E']) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - poly = Poly(degree=3) - W = Feature(('A',), 'poly(A)', poly) - U = Feature(('B',), 'poly(B)', clone(poly)) - M = ModelSpec(terms=list(D.columns.drop(['Y','C'])) + [W,U], - categorical_encoders=default_encoders, intercept=True) - MX = M.fit_transform(D) - - # each Feature gets its own fitted copy of the transform - MX = np.asarray(MX) - np.testing.assert_allclose(MX[:,M.column_map_[W]], - Poly(degree=3).fit_transform(D[['A']])) - np.testing.assert_allclose(MX[:,M.column_map_[U]], - Poly(degree=3).fit_transform(D[['B']])) - -def test_dataframe10(): - - rng = np.random.default_rng(15) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - W = Feature(('A','E'), 'AE', None) - U = Feature((W, 'C'), 'WC', None) - D['D'] = pd.Categorical(rng.choice(['a','b','c'], 50, replace=True)) - D['E'] = pd.Categorical(rng.choice(range(4,8), 50, replace=True)) - - M = ModelSpec(terms=['A', 'E', 'C', W, (W, 'D',), U], - categorical_encoders=default_encoders, intercept=True) - MX = M.fit_transform(D) - MX = np.asarray(MX) - - V = MX[:,-6:] - V2 = np.column_stack([MX[:,M.column_map_['A']], - MX[:,M.column_map_['E']], - MX[:,M.column_map_['C']]]) - np.testing.assert_allclose(V, V2) - -def test_poly_ns_bs(): - - rng = np.random.default_rng(16) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - - M = ModelSpec(terms=[poly('A', intercept=True, degree=3), - ns('E', df=5), - bs('D', df=4)], intercept=True) - - MX = M.fit_transform(D) - A = M.column_info_['A'] - M2 = ModelSpec(terms=[poly(A, intercept=True, degree=3), - ns('E', df=5), - bs('D', df=4)], intercept=True) - MX2 = M2.fit_transform(D) - # 1 intercept + 4 poly + 5 ns + 4 bs - assert MX.shape == (50, 14) - assert list(MX.columns) == list(MX2.columns) - np.testing.assert_allclose(MX, MX2) - -def test_submodel(): - - rng = np.random.default_rng(17) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - - M = ModelSpec(terms=[poly('A', intercept=True, degree=3), - ns('E', df=5), - bs('D', df=4)], intercept=True) - - M.fit(D) - MX = M.transform(D) - MXsub = M.build_submodel(D, M.terms[:2]) - # submodel is the intercept, poly and ns columns of the full model - assert list(MXsub.columns) == list(MX.columns[:10]) - np.testing.assert_allclose(MXsub, MX.iloc[:,:10]) - -def test_contrast(): - - rng = np.random.default_rng(18) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - D['C'] = pd.Categorical(rng.choice(range(4,9), 50, replace=True)) - # C has 5 levels: 'sum' and 'drop' give 4 columns, None and identity give 5 - for method, ncol in [('sum', 4), - ('drop', 4), - (None, 5), - (lambda p: np.identity(p), 5)]: - M = ModelSpec(terms=[poly('A', intercept=True, degree=3), - contrast('C', method), - bs('D', df=4)], intercept=True) - - M.fit(D) - MX = M.transform(D) - MXsub = M.build_submodel(D, M.terms[:2]) - assert MX.shape == (50, 1 + 4 + ncol + 4) - assert list(MXsub.columns) == list(MX.columns[:5 + ncol]) - np.testing.assert_allclose(MXsub, MX.iloc[:,:5 + ncol]) - - C = np.asarray(MX.iloc[:,5:5 + ncol]) - if method == 'drop': - # first level is dropped - assert np.all(C[D['C'] == 4] == 0) - elif method == 'sum': - # last level is coded as -1 in every column - assert np.all(C[D['C'] == 8] == -1) - else: - np.testing.assert_allclose(C, pd.get_dummies(D['C'])) - -def test_sequence(): - - rng = np.random.default_rng(19) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - - M = ModelSpec(terms=[poly('A', intercept=True, degree=3), - ns('E', df=5), - bs('D', df=4)], intercept=True) - M.fit(D) - MX = M.transform(D) - # each design adds one term to the previous one - seq = list(M.build_sequence(D)) - assert [df.shape[1] for df in seq] == [1, 5, 10, 14] - for df in seq: - assert list(df.columns) == list(MX.columns[:df.shape[1]]) - np.testing.assert_allclose(df, MX.iloc[:,:df.shape[1]]) - -def test_poly_ns_bs2(): - - rng = np.random.default_rng(20) - X = rng.standard_normal((50,5)) - D = pd.DataFrame(X, columns=['A','B','C','D','E']) - D['C'] = pd.Categorical(rng.choice(range(4,9), 50, replace=True)) - M = ModelSpec(terms=[(poly('A', intercept=True, degree=3),'C')], intercept=True) - MX = M.fit_transform(D) - # intercept + 4 poly columns x 4 levels of C (first level dropped) - assert MX.shape == (50, 1 + 4 * 4) - assert MX.columns[1] == 'poly(A, degree=3, intercept=True)[0]:C[5]' - P = np.asarray(Poly(degree=3, intercept=True).fit_transform(D[['A']])) - DC = np.asarray(pd.get_dummies(D['C']), float)[:,1:] - np.testing.assert_allclose(np.asarray(MX)[:,1:], - np.einsum('ij,ik->ijk', P, DC).reshape((50, -1))) - - -def test_pca(): - - rng = np.random.default_rng(21) - X = rng.standard_normal((50,8)) - D = pd.DataFrame(X, columns=['A','B','C','D','E', 'F', 'G', 'H']) - - pca_ = Feature(('A','B','C','D'), 'pca(ABCD)', PCA(n_components=2)) - M = ModelSpec(terms=[poly('F', intercept=True, degree=3), - pca_], intercept=True) - - MX = M.fit_transform(D) - - M2 = ModelSpec(terms=[poly('F', intercept=True, degree=3), - pca(['A','B','C','D'], 'ABCD', n_components=2)], intercept=True) - MX2 = M2.fit_transform(D) - - np.testing.assert_allclose(MX, MX2) - - - diff --git a/tests/models/test_model_spec.py b/tests/models/test_model_spec.py deleted file mode 100644 index 34080ba..0000000 --- a/tests/models/test_model_spec.py +++ /dev/null @@ -1,287 +0,0 @@ -# ModelSpec as used in the labs: on the ISLP datasets, and against -# statsmodels' formulas where their codings agree - -import numpy as np, pandas as pd -import pytest -import statsmodels.api as sm -import statsmodels.formula.api as smf -from sklearn.base import clone -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler - -from ISLP import load_data -from ISLP.models import (ModelSpec as MS, - contrast, - ordinal, - poly, - ns, - bs, - pca) -from ISLP.models.model_spec import Contrast -from ISLP.transforms import Poly, NaturalSpline, BSpline - -@pytest.fixture(scope='module') -def Carseats(): - return load_data('Carseats') - -@pytest.fixture(scope='module') -def Wage(): - return load_data('Wage') - -def test_matches_formula(Carseats): - - # treatment coding of categoricals and products of numeric variables - # agree with statsmodels' formulas (patsy) - - spec = MS(['Price', 'ShelveLoc', 'US', ('Income', 'Advertising')], intercept=True) - X = spec.fit_transform(Carseats) - assert list(X.columns) == ['intercept', 'Price', - 'ShelveLoc[Good]', 'ShelveLoc[Medium]', - 'US[Yes]', 'Income:Advertising'] - - fit = sm.OLS(Carseats['Sales'], X).fit() - formula = smf.ols('Sales ~ Price + ShelveLoc + US + Income:Advertising', - data=Carseats).fit() - params = formula.params.rename({'Intercept': 'intercept', - 'ShelveLoc[T.Good]': 'ShelveLoc[Good]', - 'ShelveLoc[T.Medium]': 'ShelveLoc[Medium]', - 'US[T.Yes]': 'US[Yes]'}) - np.testing.assert_allclose(fit.params, params[fit.params.index]) - np.testing.assert_allclose(fit.fittedvalues, formula.fittedvalues) - -def test_index_kept(Carseats): - - # statsmodels aligns on the index, so it must be that of the data - - D = Carseats.copy() - D.index = np.arange(1000, 1000 + D.shape[0]) - X = MS(['Price', poly('Income', degree=2), 'ShelveLoc'], intercept=True).fit_transform(D) - assert (X.index == D.index).all() - -def test_names(Carseats): - - spec = MS(['Price', poly('Income', degree=2), ('Price', 'Age')], intercept=True).fit(Carseats) - assert spec.names == ['intercept', 'Price', 'poly(Income, degree=2)', 'Price:Age'] - - spec = MS(['Price'], intercept=False).fit(Carseats) - assert spec.names == ['Price'] - assert list(spec.transform(Carseats).columns) == ['Price'] - -def test_transform_uses_fit(Carseats): - - # the transforms are fit on the training data only, and - # then applied as is to new data - - train, test = Carseats.iloc[:300], Carseats.iloc[300:] - spec = MS([poly('Price', degree=3), - ns('Income', df=4), - bs('Age', df=5)], intercept=True).fit(train) - X = np.asarray(spec.transform(test)) - - expected = np.column_stack([ - Poly(degree=3).fit(train[['Price']]).transform(test[['Price']]), - NaturalSpline(df=4).fit(train[['Income']]).transform(test[['Income']]), - BSpline(df=5).fit(train[['Age']]).transform(test[['Age']])]) - np.testing.assert_allclose(X[:,1:], expected) - - # fitting on the test data would differ - other = MS([poly('Price', degree=3), - ns('Income', df=4), - bs('Age', df=5)], intercept=True).fit(test) - assert not np.allclose(X, np.asarray(other.transform(test))) - -def test_categorical_levels_from_fit(Carseats): - - # levels missing from new data keep their columns; - # levels not seen at fit raise an error - - train, test = Carseats.iloc[:300], Carseats.iloc[300:] - spec = MS(['Price', 'ShelveLoc'], intercept=True).fit(train) - - good = test[test['ShelveLoc'] == 'Good'] - X = spec.transform(good) - assert list(X.columns) == ['intercept', 'Price', 'ShelveLoc[Good]', 'ShelveLoc[Medium]'] - assert (X['ShelveLoc[Good]'] == 1).all() - assert (X['ShelveLoc[Medium]'] == 0).all() - - new = test.copy() - new['ShelveLoc'] = new['ShelveLoc'].cat.add_categories(['Other']) - new.iloc[0, new.columns.get_loc('ShelveLoc')] = 'Other' - with pytest.raises(ValueError, match='unknown categories'): - spec.transform(new) - -@pytest.mark.parametrize('method, drop_level, columns', [ - ('drop', None, ['ShelveLoc[Good]', 'ShelveLoc[Medium]']), - ('drop', 'Medium', ['ShelveLoc[Bad]', 'ShelveLoc[Good]']), - ('sum', None, ['ShelveLoc[Bad]', 'ShelveLoc[Good]']), - ('sum', 'Bad', ['ShelveLoc[Good]', 'ShelveLoc[Medium]']), - (None, None, ['ShelveLoc[Bad]', 'ShelveLoc[Good]', 'ShelveLoc[Medium]'])]) -def test_contrast_methods(Carseats, method, drop_level, columns): - - X = MS([contrast('ShelveLoc', method, drop_level)], intercept=True).fit_transform(Carseats) - assert list(X.columns) == ['intercept'] + columns - level = np.asarray(Carseats['ShelveLoc']) - C = np.asarray(X[columns]) - if method is None: - for j, l in enumerate(['Bad', 'Good', 'Medium']): - np.testing.assert_allclose(C[:,j], level == l) - return - if drop_level is None: - # as R: 'drop' drops the first level, 'sum' codes the last as -1 - drop_level = 'Bad' if method == 'drop' else 'Medium' - for j, c in enumerate(columns): - l = c[len('ShelveLoc['):-1] - np.testing.assert_allclose(C[level == l, j], 1) - np.testing.assert_allclose(C[level == drop_level], 0 if method == 'drop' else -1) - -def test_contrast_of_column(Carseats): - - # contrast() accepts a Column, e.g. from column_info_ - col = MS(['ShelveLoc'], intercept=True).fit(Carseats).column_info_['ShelveLoc'] - X = MS([contrast(col, 'sum')], intercept=True).fit_transform(Carseats) - assert list(X.columns) == ['intercept', 'ShelveLoc[Bad]', 'ShelveLoc[Good]'] - -def test_contrast_bad_method(Carseats): - - with pytest.raises(ValueError, match='method must be one of'): - MS([contrast('ShelveLoc', 'bogus')], intercept=True).fit(Carseats) - -def test_ordered_categorical(Wage): - - # an ordered categorical is dummy coded, in the order of its - # categories; ordinal() gives its integer codes - - assert Wage['education'].dtype.ordered - X = MS(['education'], intercept=True).fit_transform(Wage) - levels = list(Wage['education'].cat.categories) - assert list(X.columns) == ['intercept'] + [f'education[{l}]' for l in levels[1:]] - np.testing.assert_allclose(X.iloc[:, 1:], pd.get_dummies(Wage['education']).iloc[:, 1:]) - - X = MS([ordinal('education')], intercept=True).fit_transform(Wage) - assert list(X.columns) == ['intercept', 'ordinal(education)'] - np.testing.assert_allclose(X['ordinal(education)'], Wage['education'].cat.codes) - - # an unordered categorical is dummy coded - X = MS(['jobclass'], intercept=True).fit_transform(Wage) - assert list(X.columns) == ['intercept', 'jobclass[2. Information]'] - -def test_pca_scale(Carseats): - - cols = ['Price', 'Income', 'Advertising', 'Age'] - X = MS([pca(cols, 'P', n_components=2, scale=True)], - intercept=False).fit_transform(Carseats) - Z = StandardScaler().fit_transform(Carseats[cols]) - expected = PCA(n_components=2).fit_transform(Z) - np.testing.assert_allclose(np.asarray(X), expected) - -def test_build_sequence_drop(Carseats): - - spec = MS(['Price', 'Income', 'Age'], intercept=True).fit(Carseats) - X = spec.transform(Carseats) - seq = list(spec.build_sequence(Carseats, anova_type='drop')) - # each design drops one term (the intercept first) - assert [list(D.columns) for D in seq] == [['Price', 'Income', 'Age'], - ['intercept', 'Income', 'Age'], - ['intercept', 'Price', 'Age'], - ['intercept', 'Price', 'Income']] - for D in seq: - np.testing.assert_allclose(D, X[D.columns]) - - with pytest.raises(ValueError, match='anova_type'): - list(spec.build_sequence(Carseats, anova_type='bogus')) - -def test_ndarray(Carseats): - - # an ndarray gives ndarray designs - X = np.asarray(Carseats[['Price', 'Income']]) - spec = MS([0, 1, (0, 1)], intercept=True).fit(X) - M = spec.transform(X) - assert isinstance(M, np.ndarray) - np.testing.assert_allclose(M, np.column_stack([np.ones(X.shape[0]), - X, - X[:,0] * X[:,1]])) - assert [D.shape[1] for D in spec.build_sequence(X)] == [1, 2, 3, 4] - -def test_clone_is_unfitted(Carseats): - - spec = MS([poly('Price', degree=2)], intercept=True) - fitted = clone(spec).fit(Carseats) - assert hasattr(fitted, 'column_info_') - assert not hasattr(clone(spec), 'column_info_') - # clones of an unfitted spec are fit independently - before = np.asarray(fitted.transform(Carseats)) - clone(spec).fit(Carseats.iloc[:100]) - np.testing.assert_allclose(fitted.transform(Carseats), before) - -def test_categorical_encoders(Carseats, Wage): - - # 'categorical' sets the encoder of every unordered categorical column - X = MS(['ShelveLoc', 'US'], - categorical_encoders={'categorical': Contrast(method=None)}, intercept=True).fit_transform(Carseats) - assert list(X.columns) == ['intercept', 'ShelveLoc[Bad]', 'ShelveLoc[Good]', - 'ShelveLoc[Medium]', 'US[No]', 'US[Yes]'] - - # a column name sets the encoder of that column only - X = MS(['ShelveLoc', 'US'], - categorical_encoders={'ShelveLoc': Contrast(method='sum')}, intercept=True).fit_transform(Carseats) - assert list(X.columns) == ['intercept', 'ShelveLoc[Bad]', 'ShelveLoc[Good]', 'US[Yes]'] - - # 'ordinal' sets the encoder of the ordered categorical columns - X = MS(['education'], - categorical_encoders={'ordinal': Contrast(method='drop')}, intercept=True).fit_transform(Wage) - assert X.shape[1] == 1 + 4 - assert X.columns[1] == 'education[2. HS Grad]' - -def test_categorical_encoders_set_params(Carseats): - - # categorical_encoders was merged in __init__, so set_params - # (as in GridSearchCV) was ignored - spec = MS(['ShelveLoc'], intercept=True) - spec.set_params(categorical_encoders={'categorical': Contrast(method=None)}) - X = spec.fit_transform(Carseats) - assert list(X.columns) == ['intercept', 'ShelveLoc[Bad]', 'ShelveLoc[Good]', 'ShelveLoc[Medium]'] - -def test_ndarray_categorical(Carseats): - - # categorical columns of an ndarray, and an integer key in - # categorical_encoders (both raised TypeError) - codes = np.asarray(Carseats['ShelveLoc'].cat.codes) - X = np.column_stack([Carseats['Price'], codes]) - - spec = MS([0, 1], categorical_features=[False, True], intercept=True).fit(X) - M = spec.transform(X) - np.testing.assert_allclose(M, np.column_stack([np.ones(X.shape[0]), - X[:,0], - codes == 1, - codes == 2])) - np.testing.assert_array_equal(spec.known_categories_[1], [0, 1, 2]) - - M = MS([0, 1], categorical_features=[False, True], - categorical_encoders={1: Contrast(method=None)}, intercept=True).fit_transform(X) - assert M.shape == (X.shape[0], 1 + 1 + 3) - -def test_known_categories(Carseats): - - spec = MS(['Price', 'ShelveLoc'], intercept=True).fit(Carseats) - known = dict(zip(Carseats.columns, spec.known_categories_)) - assert sorted(known['ShelveLoc']) == ['Bad', 'Good', 'Medium'] - assert known['Price'] is None - -# fit fits clones of the encoders of the terms (e.g. the Poly of poly()), -# so a fitted spec, or a clone of it, can be fit again - -def test_refit(Carseats): - spec = MS([poly('Price', degree=2)], intercept=True).fit(Carseats.iloc[:200]) - spec.fit(Carseats.iloc[200:]) - -def test_clone_fitted_then_fit(Carseats): - spec = MS([poly('Price', degree=2)], intercept=True).fit(Carseats.iloc[:200]) - clone(spec).fit(Carseats.iloc[200:]) - -def test_refit_contrast(Carseats): - # the Contrast of contrast() is kept from the first fit, so a level - # that only the second fit sees is unknown at transform - spec = MS([contrast('ShelveLoc', None)], intercept=True).fit(Carseats[Carseats['ShelveLoc'] != 'Bad'].assign( - ShelveLoc=lambda df: df['ShelveLoc'].cat.remove_unused_categories())) - X = spec.fit(Carseats).transform(Carseats) - assert X.shape[1] == 1 + 3 diff --git a/tests/models/test_model_spec_hard.py b/tests/models/test_model_spec_hard.py deleted file mode 100644 index 281ce9d..0000000 --- a/tests/models/test_model_spec_hard.py +++ /dev/null @@ -1,1136 +0,0 @@ -""" -Adversarial tests for ModelSpec and the transforms it relies on. - -Tests marked ``xfail(strict=True)`` document known bugs: once a bug is -fixed the test will XPASS and fail the suite, signalling that the -marker should be removed. -""" -import pickle -from copy import deepcopy - -import numpy as np, pandas as pd -import pytest - -from sklearn.base import clone -from sklearn.pipeline import make_pipeline -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import cross_val_score, GridSearchCV -from sklearn.utils.validation import check_is_fitted -from sklearn.exceptions import NotFittedError -import statsmodels.api as sm - -from ISLP import load_data -from ISLP.models import (ModelSpec as MS, - ordinal, - poly, - ns, - bs, - pca, - contrast, - derived_feature, - sklearn_sm) -from ISLP.models.model_spec import Contrast, Ordinal -from ISLP.transforms import Poly, BSpline, NaturalSpline - -bug = lambda reason: pytest.mark.xfail(strict=True, reason=reason) - -# ModelSpec warns at fit that it moves to its own package in ISLP 0.7; these -# tests use its default intercept (True), which then becomes False, when -# those relying on it must change -pytestmark = pytest.mark.filterwarnings("ignore:ModelSpec will move:FutureWarning") - -@pytest.fixture -def D(): - rng = np.random.default_rng(0) - n = 40 - return pd.DataFrame({'x': rng.uniform(0, 10, n), - 'z': rng.standard_normal(n), - 'g': pd.Categorical(rng.choice(['a', 'b', 'c'], n)), - 'o': pd.Categorical(rng.choice(['lo', 'mid', 'hi'], n), - categories=['lo', 'mid', 'hi'], - ordered=True), - 'b': rng.choice([True, False], n)}) - -# reference values from R: library(splines) -# x <- c(0.5,1.3,2,2.2,3.7,4.1,5,6.6,7,9.4); xn <- c(-1,10,12) - -R_x = np.array([0.5, 1.3, 2, 2.2, 3.7, 4.1, 5, 6.6, 7, 9.4]) -R_xn = np.array([-1, 10, 12.]) - -# poly(x, 3); predict(poly(x, 3), xn) -R_poly3 = np.array([ - [-0.432265634900399, 0.480076800172643, -0.492074588241283], - [-0.33829484470466, 0.202199643711212, 0.0288590978338659], - [-0.256070403283389, 0.00601813290167871, 0.269254002877709], - [-0.232577705734454, -0.0419832699991788, 0.306657205224392], - [-0.0563824741174433, -0.287945646241078, 0.257001756602356], - [-0.00939707901957392, -0.319544793399292, 0.174814373415067], - [0.0963200599506324, -0.33831490185743, -0.0557761335361995], - [0.28426164034211, -0.192784931773284, -0.407723859127603], - [0.33124703543998, -0.120622628894747, -0.44128781443095], - [0.613159406027197, 0.612901595379477, 0.360275959382646]]) -R_poly3_new = np.array([ - [-0.60846086651741, 1.15539690070455, -2.36769085138778], - [0.683637498674001, 0.876787224752409, 0.942282386603316], - [0.918564474163348, 1.9889747566526, 4.46468148147144]]) - -# ns(x, df=4); predict(ns(x, df=4), xn) -R_ns4 = np.array([ - [0, 0, 0, 0], - [0.0170445088052199, -0.126976258270368, 0.345216702172563, -0.218240443902195], - [0.112353939878158, -0.189074505657504, 0.514046312256339, -0.324971806598835], - [0.163210626930167, -0.192299563791128, 0.522977044802782, -0.330617672001759], - [0.633757240749562, -0.00854698697500502, 0.239665368301396, -0.151512589156055], - [0.677452961078211, 0.0905991512403247, 0.167628785520828, -0.105899529804294], - [0.570168367862708, 0.341126610235441, 0.121936366947747, -0.0649922559664666], - [0.16969696969697, 0.568006802401707, 0.229595142333996, 0.0327010855673276], - [0.106864564007421, 0.535077582575308, 0.25926524352366, 0.0987926098936108], - [0, -0.151489068686521, 0.411860905491479, 0.739628163195042]]) -R_ns4_new = np.array([ - [0, 0.257561121712616, -0.700244299656176, 0.442683177943559], - [0, -0.3714925116199, 0.447495265966603, 0.923997245653297], - [0, -1.10483732139783, 0.566276467550352, 1.53856085384748]]) - -# bs(x, df=5) -R_bs5 = np.array([ - [0, 0, 0, 0, 0], - [0.654880149163343, 0.189217994999522, 0.00752001175001836, 0, 0], - [0.473573512449963, 0.475227764602206, 0.0495703899537343, 0, 0], - [0.38716049382716, 0.540679705923152, 0.0721598002496879, 0, 0], - [0.0387477954144621, 0.578448343340665, 0.359552373149634, 0.0232514880952381, 0], - [0.0128571428571429, 0.502016006777243, 0.437872937226285, 0.0472539131393298, 0], - [0, 0.302122347066167, 0.546643085032598, 0.151234567901235, 0], - [0, 0.0778572239246397, 0.410842908714887, 0.463215720102772, 0.0480841472577009], - [0, 0.0490296220633299, 0.336645309066147, 0.520410718757826, 0.0939143501126972], - [0, 0, 0, 0, 1]]) - -#### numerics: agreement with R - -def test_poly_matches_R(): - P = Poly(degree=3).fit(R_x) - np.testing.assert_allclose(P.transform(R_x), R_poly3, atol=1e-12) - np.testing.assert_allclose(P.transform(R_xn), R_poly3_new, atol=1e-12) - -def test_ns_matches_R(): - N = NaturalSpline(df=4).fit(R_x) - np.testing.assert_allclose(N.transform(R_x), R_ns4, atol=1e-12) - np.testing.assert_allclose(N.transform(R_xn), R_ns4_new, atol=1e-12) - -def test_bs_matches_R(): - np.testing.assert_allclose(BSpline(df=5).fit_transform(R_x), R_bs5, atol=1e-12) - -def test_ns_through_modelspec_matches_R(): - df = pd.DataFrame({'x': R_x}) - M = MS([ns('x', df=4)], intercept=False).fit(df) - np.testing.assert_allclose(M.transform(pd.DataFrame({'x': R_xn})), R_ns4_new, atol=1e-12) - -def test_bspline_partition_of_unity(): - x = np.linspace(0, 1, 50) - B = BSpline(df=7, intercept=True).fit_transform(x) - np.testing.assert_allclose(B.sum(1), 1) - -def test_ns_linear_beyond_boundary(): - x = np.linspace(0, 1, 50) - N = NaturalSpline(df=5, intercept=True).fit(x) - V = N.transform(np.linspace(1.5, 4, 10)) - np.testing.assert_allclose(np.diff(V, 2, axis=0), 0, atol=1e-10) - -#### invariants that currently hold - -def test_column_map_matches_names(D): - M = MS(['x', 'g', poly('z', 3), ('x', 'g'), ('g', 'o')]).fit(D) - X = M.transform(D) - for term, slc in M.column_map_.items(): - if term != 'intercept': - assert list(X.columns[slc]) == list(M.column_names_[term]) - assert X.shape[1] == max(s.stop for s in M.column_map_.values()) - -def test_index_preserved(D): - D.index = np.arange(100, 100 + D.shape[0]) - X = MS(['x', poly('z', 2), 'g', ('x', 'g'), ns('x', df=3)]).fit_transform(D) - assert (X.index == D.index).all() - assert not X.isna().any().any() - -def test_transform_ignores_column_order(D): - M = MS(['x', 'z', 'g', ('x', 'g')]).fit(D) - np.testing.assert_allclose(M.transform(D), M.transform(D[D.columns[::-1]])) - -def test_unused_columns_not_needed_at_transform(D): - M = MS(['x']).fit(D) - assert M.transform(D[['x']]).shape == (D.shape[0], 2) - -def test_pickle_roundtrip(D): - M = MS(['x', ns('z', df=3), 'g', pca(['x', 'z'], 'pc', n_components=1)]).fit(D) - M2 = pickle.loads(pickle.dumps(M)) - np.testing.assert_allclose(M.transform(D), M2.transform(D)) - -def test_predict_categories_as_strings(D): - M = MS(['g']).fit(D) - np.testing.assert_allclose(M.transform(D), M.transform(D.assign(g=D.g.astype(str)))) - -def test_bool_column_is_float(D): - X = MS(['b']).fit_transform(D) - assert (X.dtypes == float).all() - -def test_sum_contrast(D): - X = MS([contrast('g', 'sum')], intercept=False).fit_transform(D) - np.testing.assert_allclose(X[D.g == 'c'], -1) - -def test_cv_pipeline_unfitted_spec(D): - y = D.x + D.z - pipe = make_pipeline(MS([poly('x', 3), 'g', ns('z', df=3)]), LinearRegression()) - cross_val_score(pipe, D, y, cv=4, error_score='raise') - -def test_gridsearch_over_spec_param(D): - pipe = make_pipeline(MS([ns('x', df=4)]), LinearRegression()) - GridSearchCV(pipe, {'modelspec__intercept': [True, False]}, cv=3, - error_score='raise').fit(D, D.z) - -#### refitting: encoders are fit on clones, `terms` is never mutated - -# a spec touching every kind of term that carries an encoder -def _rich_terms(): - p = poly('x', 2) - return ['x', - p, - ns('z', df=3), - bs('x', df=4), - contrast('g', 'sum'), - (p, 'g'), - (bs('z', df=3), 'x'), - pca(['x', 'z'], 'pc', scale=True, n_components=1)] - -def _shifted(D): - D2 = D.assign(x=D.x * 3 + 100, z=D.z * 5 - 2) - D2['g'] = pd.Categorical(D.g.astype(str).replace('a', 'c')) - return D2 - -def _assert_same_design(A, B): - assert list(A.columns) == list(B.columns) - np.testing.assert_allclose(A, B) - -def test_refit_matches_fresh_fit(D): - D2 = _shifted(D) - M = MS(_rich_terms()).fit(D) - M.fit(D2) - _assert_same_design(M.transform(D2), MS(_rich_terms()).fit_transform(D2)) - -def test_clone_fitted_then_fit(D): - D2 = _shifted(D) - M = MS(_rich_terms()).fit(D) - A = M.transform(D) - _assert_same_design(clone(M).fit_transform(D2), MS(_rich_terms()).fit_transform(D2)) - _assert_same_design(M.transform(D), A) # original unaffected - -def test_shared_terms_two_specs(D): - D2 = _shifted(D) - terms = _rich_terms() - M1 = MS(terms).fit(D) - A = M1.transform(D) - M2 = MS(terms).fit(D2) - _assert_same_design(M1.transform(D), A) # fitting M2 leaves M1 alone - _assert_same_design(M2.transform(D2), MS(_rich_terms()).fit_transform(D2)) - -def test_terms_not_mutated(D): - terms = _rich_terms() - MS(terms).fit(D) - for t in terms: - if hasattr(t, 'encoder') and t.encoder is not None: - with pytest.raises(NotFittedError): - check_is_fitted(t.encoder) - -def test_categorical_encoder_not_mutated(D): - enc = Contrast(method=None) - MS(['g'], categorical_encoders={'g': enc}).fit(D) - with pytest.raises(NotFittedError): - check_is_fitted(enc) - -def test_shared_feature_fit_once(D): - # poly('x') used as a term and inside an interaction: - # the interaction must use the same fitted copy - p = poly('x', 2) - X = MS([p, (p, 'z')], intercept=False).fit_transform(D) - np.testing.assert_allclose(X.iloc[:, 2:], X.iloc[:, :2].values * D.z.values[:, None]) - -def test_cv_prefitted_spec(D): - M = MS([poly('x', 3), 'g']).fit(D) - cross_val_score(sklearn_sm(sm.OLS, M), D, D.z, cv=4, error_score='raise') - -def test_cv_folds_refit_encoders(D): - # each fold's design must be fit on that fold alone - from sklearn.model_selection import KFold - spec = MS([ns('x', df=4)]) - for train, test in KFold(4).split(D): - M = clone(spec).fit(D.iloc[train]) - knots = M.terms_[0].encoder.boundary_knots_ - assert knots == [D.x.iloc[train].min(), D.x.iloc[train].max()] - -def test_build_submodel_accepts_original_terms(D): - terms = _rich_terms() - M = MS(terms).fit(D) - X = M.transform(D) - sub = M.build_submodel(D, terms[1:3]) - _assert_same_design(sub, X[['intercept'] + M.column_names_[terms[1]] + M.column_names_[terms[2]]]) - _assert_same_design(M.build_submodel(D, M.terms_[1:3]), sub) - -def test_build_submodel_copied_feature_is_unfitted(D): - # a copy of a term is a different (unfitted) term - f = poly('x', 2) - M = MS([f]).fit(D) - with pytest.raises(NotFittedError): - M.build_submodel(D, [deepcopy(f)]) - -def test_pickle_refit(D): - M = pickle.loads(pickle.dumps(MS(_rich_terms()).fit(D))) - D2 = _shifted(D) - _assert_same_design(M.fit_transform(D2), MS(_rich_terms()).fit_transform(D2)) - -def test_bspline_does_not_mutate_params(): - B = BSpline(df=4).fit(np.linspace(0, 1, 20)) - assert B.lower_bound is None and B.upper_bound is None - -def test_bspline_refit_bounds(): - B = BSpline(df=4) - B.fit(np.linspace(0, 1, 20)) - B.fit(np.linspace(0, 100, 20)) - assert B.boundary_knots_ == [0, 100] - -def test_contrast_bad_drop_level_message(D): - with pytest.raises(ValueError): - MS([contrast('g', 'drop', 'zzz')]).fit_transform(D) - -#### frozen encoders: fit elsewhere, used as is - -@pytest.fixture -def frozen(D): - # models fit on other data, frozen so that ModelSpec.fit does not refit them; - # they are fit with the column names ModelSpec will pass them - FrozenEstimator = pytest.importorskip('sklearn.frozen').FrozenEstimator - from sklearn.decomposition import PCA - from sklearn.neighbors import KNeighborsRegressor - from ISLP.models import derived_feature - other = D.assign(x=D.x * 2 + 1, z=D.z + 3) - XZ = other[['x', 'z']] - knn = KNeighborsRegressor(5).fit(XZ, other.x + other.z) - pc = PCA(n_components=1).fit(XZ) - terms = [derived_feature(['x', 'z'], encoder=FrozenEstimator(knn), - name='knn', use_transform=False), - derived_feature(['x', 'z'], encoder=FrozenEstimator(pc), name='pc')] - return knn, pc, terms - -def test_frozen_used_as_is(D, frozen): - knn, pc, terms = frozen - X = MS(terms, intercept=False).fit_transform(D) - XZ = D[['x', 'z']] - np.testing.assert_allclose(X['knn'], knn.predict(XZ)) - np.testing.assert_allclose(X['pc'], pc.transform(XZ)[:, 0]) - -def test_frozen_survives_refit_clone_cv_pickle(D, frozen): - knn, pc, terms = frozen - components = pc.components_.copy() - expected = MS(terms + ['g']).fit_transform(D) - - M = MS(terms + ['g']).fit(_shifted(D)) - M.fit(D) # refit: the same frozen objects - assert M.terms_[0].encoder.estimator is knn - assert M.terms_[1].encoder.estimator is pc - _assert_same_design(M.transform(D), expected) - - # clone() deep-copies `terms`, and pickling copies everything, so - # these hold copies of the frozen models: compare values - for est in [clone(M).fit(_shifted(D)).fit(D), - pickle.loads(pickle.dumps(M)).fit(_shifted(D)).fit(D)]: - _assert_same_design(est.transform(D), expected) - - cross_val_score(make_pipeline(MS(terms + ['g']), LinearRegression()), - D, D.z, cv=4, error_score='raise') - np.testing.assert_allclose(pc.components_, components) # never refit - -def test_frozen_unfitted_raises(D): - FrozenEstimator = pytest.importorskip('sklearn.frozen').FrozenEstimator - from sklearn.decomposition import PCA - from ISLP.models import derived_feature - f = derived_feature(['x', 'z'], encoder=FrozenEstimator(PCA(1)), name='pc') - # freezing an unfitted model is an error: ModelSpec tries to fit it - with pytest.raises((NotFittedError, TypeError)): - MS([f]).fit_transform(D) - -def test_prefit_unfrozen_encoder_is_refit(D): - # a fitted (but not frozen) encoder is cloned, so refit on the spec's data - from sklearn.decomposition import PCA - from ISLP.models import derived_feature - pc = PCA(n_components=1).fit(D[['x', 'z']].values * 10) - components = pc.components_.copy() - M = MS([derived_feature(['x', 'z'], encoder=pc, name='pc')]).fit(D) - np.testing.assert_allclose(M.terms_[0].encoder.components_, - PCA(n_components=1).fit(D[['x', 'z']].values).components_) - np.testing.assert_allclose(pc.components_, components) # original untouched - -#### known bugs - -# strategy.py: the ModelSpec <-> selection boundary - -@bug('_postprocess_best reduces state to names; build_submodel cannot resolve derived names') -@pytest.mark.parametrize('term', [poly('z', 2), ('x', 'z')]) -def test_stepwise_with_derived_term(D, term): - from ISLP.models import Stepwise, sklearn_selected - M = MS(['x', term, 'g']).fit(D) - strategy = Stepwise.first_peak(M, direction='forward', max_terms=3) - sklearn_selected(sm.OLS, strategy).fit(D, D.x + D.z) - -def test_best_1sd_picks_smaller_model(): - from ISLP.models.strategy import _postprocess_best_1sd - class T(str): - name = property(str) - results = [((T('a'),), 0, np.array([-1.0, -1.1, -0.9])), - ((T('a'), T('b')), 1, np.array([-0.98, -0.99, -0.97]))] - assert _postprocess_best_1sd(results)[0] == ('a',) - -# sklearn API conformance - -def test_no_attributes_set_in_init(): - M = MS(['x'], categorical_encoders={'categorical': Contrast(method=None)}) - assert set(vars(M)) == set(M.get_params()) - -def test_default_params(): - assert MS().get_params() == {'terms': (), 'intercept': None, - 'categorical_features': None, - 'categorical_encoders': None, - 'sparse_output': False} - -def test_unfitted_raises_notfitted(D): - M = MS(['x']) - with pytest.raises(NotFittedError): - check_is_fitted(M) - for method in [lambda: M.transform(D), - lambda: M.get_feature_names_out(), - lambda: M.build_submodel(D, ['x']), - lambda: M.build_sequence(D)]: - with pytest.raises(NotFittedError): - method() - -def test_set_params_categorical_encoders(D): - M = MS(['g']).set_params(categorical_encoders={'categorical': Contrast(method=None)}) - assert list(M.fit_transform(D).columns) == ['intercept', 'g[a]', 'g[b]', 'g[c]'] - # and a per-column encoder - M.set_params(categorical_encoders={'g': Contrast(method='sum')}) - assert list(M.fit_transform(D).columns) == ['intercept', 'g[a]', 'g[b]'] - -def test_categorical_encoders_dict_not_modified(D): - encoders = {'g': Contrast(method=None)} - MS(['g'], categorical_encoders=encoders).fit(D) - assert list(encoders) == ['g'] - -def test_gridsearch_over_categorical_encoders(D): - pipe = make_pipeline(MS(['x', 'g']), LinearRegression()) - grid = {'modelspec__categorical_encoders': [None, {'categorical': Contrast(method='sum')}]} - G = GridSearchCV(pipe, grid, cv=3, error_score='raise').fit(D, D.z) - assert len(G.cv_results_['params']) == 2 - -def test_get_feature_names_out(D): - M = MS(['x', 'g']).fit(D) - assert list(M.get_feature_names_out()) == ['intercept', 'x', 'g[b]', 'g[c]'] - -@pytest.mark.parametrize('intercept', [True, False]) -def test_feature_names_match_transform(D, intercept): - M = MS(_rich_terms(), intercept=intercept).fit(D) - assert list(M.get_feature_names_out()) == list(M.transform(D).columns) - assert list(MS([], intercept=intercept).fit(D).get_feature_names_out()) == \ - list(MS([], intercept=intercept).fit_transform(D).columns) - -def test_n_features_in(D): - M = MS(['x', 'g']).fit(D) - assert M.n_features_in_ == D.shape[1] - assert list(M.feature_names_in_) == list(D.columns) - A = D[['x', 'z']].values - M = MS([0, (0, 1)]).fit(A) - assert M.n_features_in_ == 2 and not hasattr(M, 'feature_names_in_') - with pytest.raises(ValueError, match='X has 3 features, but ModelSpec is expecting 2'): - M.transform(np.column_stack([A, A[:, 0]])) - -def test_set_output_pandas_on_ndarray(D): - A = D[['x', 'z']].values - X = MS([0, poly(1, 2)]).set_output(transform='pandas').fit_transform(A) - assert isinstance(X, pd.DataFrame) - assert list(X.columns) == ['intercept', '0', 'poly(1, degree=2)[0]', 'poly(1, degree=2)[1]'] - np.testing.assert_allclose(X, MS([0, poly(1, 2)]).fit_transform(A)) - -def test_pipeline_set_output(D): - pipe = make_pipeline(MS(['x', 'g', ns('z', df=3)]), LinearRegression()).set_output(transform='pandas') - pipe.fit(D, D.z) - assert list(pipe[:-1].get_feature_names_out()) == list(pipe[0].transform(D).columns) - -@pytest.mark.parametrize('bad, exc, match', [ - (np.arange(5.), ValueError, 'Reshape your data'), - (np.zeros((0, 2)), ValueError, '0 sample'), - (np.zeros((3, 0)), ValueError, '0 feature'), - (np.ones((3, 2)) * 1j, ValueError, 'Complex'), -]) -def test_bad_input(bad, exc, match): - with pytest.raises(exc, match=match): - MS([0]).fit(bad) - -def test_sparse_input_raises(): - import scipy.sparse - with pytest.raises(TypeError, match='Sparse data is not supported'): - MS([0]).fit(scipy.sparse.csr_matrix(np.eye(3))) - -def test_array_like_input(D): - A = D[['x', 'z']].values - np.testing.assert_allclose(MS([0, 1]).fit_transform(A.tolist()), MS([0, 1]).fit_transform(A)) - -def test_dataframe_extra_columns_at_transform(D): - M = MS(['x', 'g']).fit(D[['x', 'g']]) - np.testing.assert_allclose(M.transform(D), M.transform(D[['x', 'g']])) - -def test_column_get_columns_requires_fit(D): - # a Column in the terms of a ModelSpec is not fit itself (a copy is) - shelve = contrast('g', None) - MS([shelve]).fit(D) - with pytest.raises(NotFittedError, match=r"Column 'g' is not fit.*fit=True"): - shelve.get_columns(D) - cols, names = shelve.get_columns(D, fit=True) - assert names == ['g[a]', 'g[b]', 'g[c]'] - np.testing.assert_allclose(cols, _indicators(D.g, ['a', 'b', 'c'])) - np.testing.assert_allclose(shelve.get_columns(D)[0], cols) # now fit - -def test_column_stateless_encoder(D): - # an encoder with nothing to fit is used without checks - from ISLP.models.columns import Column - from sklearn.base import BaseEstimator, TransformerMixin - class Double(TransformerMixin, BaseEstimator): - def fit(self, X, y=None): - return self - def transform(self, X): - return 2 * np.asarray(X, float) - cols, _ = Column('x', 'x', encoder=Double()).get_columns(D) - np.testing.assert_allclose(cols[:, 0], 2 * D.x) - -@pytest.mark.parametrize('klass, col', [(Contrast, 'g'), (Ordinal, 'o')]) -def test_single_column_encoders(D, klass, col): - with pytest.raises(NotFittedError): - klass().transform(D[[col]]) - with pytest.raises(ValueError, match='single column'): - klass().fit(D[['g', 'o']]) - with pytest.raises(ValueError, match='single column'): - klass().fit(np.column_stack([D[col], D[col]])) - E = klass().fit(D[[col]]) - assert E.n_features_in_ == 1 - np.testing.assert_allclose(E.transform(D[[col]]), E.transform(D[col])) - -# input handling - -@pytest.mark.parametrize('term', [poly('x', 3), ns('x', df=4), bs('x', df=4)]) -def test_single_row_predict(D, term): - # one row gives that row of the model matrix, also outside the range of x - M = MS([term, 'g'], intercept=True).fit(D) - new = pd.concat([D.iloc[[3]], D.iloc[[5]].assign(x=D.x.max() + 2)]) - full = M.transform(new) - for i in range(new.shape[0]): - one = M.transform(new.iloc[[i]]) - assert one.shape == (1, full.shape[1]) - np.testing.assert_allclose(one.values[0], full.values[i]) - # the same term on column 0 of an np.ndarray - helper, _, kwargs = term.spec - N = MS([{'poly': poly, 'ns': ns, 'bs': bs}[helper](0, **dict(kwargs))], - intercept=False) - A = D[['x', 'z']].values - N.fit(A) - np.testing.assert_allclose(N.transform(A[[7]]), N.transform(A)[[7]]) - -@pytest.mark.parametrize('klass, args', [(Poly, {'degree': 3}), (NaturalSpline, {'df': 4}), - (BSpline, {'df': 4})]) -def test_transform_single_value(klass, args): - x = np.linspace(0, 1, 20) - T = klass(**args).fit(x) - for one in (0.3, np.array([0.3]), np.array([[0.3]]), pd.Series([0.3])): - np.testing.assert_allclose(np.asarray(T.transform(one)).reshape(-1), - np.asarray(T.transform(np.array([0.3, 0.5])))[0]) - with pytest.raises(ValueError, match='single column'): - T.transform(np.ones((3, 2))) - -def test_ndarray_categorical_features(): - rng = np.random.default_rng(1) - A = np.column_stack([rng.standard_normal(30), rng.choice([0, 1, 2], 30)]) - assert MS([0, 1], categorical_features=[1]).fit_transform(A).shape == (30, 4) - -def test_series_input(D): - _assert_same_design(MS(['x']).fit_transform(D.x), MS(['x']).fit_transform(D[['x']])) - -@bug('fit on ndarray then transform DataFrame fails (int labels via .loc)') -def test_fit_ndarray_transform_df(D): - A = D[['x', 'z']] - MS([0, 1]).fit(A.values).transform(A) - -# categoricals - -def _indicators(x, levels): - return np.column_stack([(np.asarray(x) == l).astype(float) for l in levels]) - -@pytest.mark.parametrize('method', ['drop', 'sum', None]) -def test_unused_category_level(D, method): - # unused categories are dropped, as in R's lm (drop.unused.levels) - Du = D.assign(g=pd.Categorical(D.g.astype(str), categories=['a', 'd', 'b', 'c'])) - _assert_same_design(MS([contrast('g', method)]).fit_transform(Du), - MS([contrast('g', method)]).fit_transform(D)) - -def test_train_missing_level(D): - Dt = D[D.g != 'c'] # categories still a, b, c - M = MS(['g']).fit(Dt) - assert list(M.transform(Dt).columns) == ['intercept', 'g[b]'] - with pytest.raises(ValueError, match="unknown categories \\['c'\\]"): - M.transform(D) # as in R, a new level cannot be predicted - -def test_cv_fold_missing_level(D): - # a level absent from some training folds; each fold's design is - # built from the levels it saw - Dr = D.copy() - g = Dr.g.astype(object) - g.iloc[:5] = 'rare' - Dr['g'] = pd.Categorical(g, categories=['a', 'b', 'c', 'rare']) - pipe = make_pipeline(MS(['x', 'g']), LinearRegression()) - from sklearn.model_selection import KFold - for train, _ in KFold(4).split(Dr): - pipe.fit(Dr.iloc[train], Dr.z.iloc[train]) - nlev = Dr.g.iloc[train].nunique() - assert pipe[0].transform(Dr.iloc[train]).shape[1] == 2 + nlev - 1 - -def test_nan_categorical_propagates(D): - g = D.g.astype(object) - g.iloc[:3] = np.nan - Dn = D.assign(g=pd.Categorical(g)) - for method in ['drop', 'sum', None]: - X = MS([contrast('g', method)], intercept=False).fit_transform(Dn) - assert X.iloc[:3].isna().all().all() - assert not X.iloc[3:].isna().any().any() - _assert_same_design(X.iloc[3:], MS([contrast('g', method)], intercept=False).fit_transform(Dn.iloc[3:])) - -def test_contrast_values(D): - levels = ['a', 'b', 'c'] - I = _indicators(D.g, levels) - np.testing.assert_allclose(MS([contrast('g', None)], intercept=False).fit_transform(D), I) - np.testing.assert_allclose(MS([contrast('g', 'drop')], intercept=False).fit_transform(D), I[:, 1:]) - np.testing.assert_allclose(MS([contrast('g', 'drop', 'b')], intercept=False).fit_transform(D), I[:, [0, 2]]) - np.testing.assert_allclose(MS([contrast('g', 'sum', 'a')], intercept=False).fit_transform(D), - I[:, 1:] - I[:, [0]]) - -def test_contrast_follows_category_order(D): - # all methods use the order of dtype.categories, not sorted order - Dr = D.assign(g=pd.Categorical(D.g.astype(str), categories=['c', 'a', 'b'])) - I = _indicators(Dr.g, ['c', 'a', 'b']) - X = MS([contrast('g', None)], intercept=False).fit_transform(Dr) - assert list(X.columns) == ['g[c]', 'g[a]', 'g[b]'] - np.testing.assert_allclose(X, I) - np.testing.assert_allclose(MS([contrast('g', 'drop')], intercept=False).fit_transform(Dr), I[:, 1:]) - C = np.array([[1, 0], [0, 1], [-1, -1.]]) - np.testing.assert_allclose(MS([contrast('g', lambda p: C)], intercept=False).fit_transform(Dr), I @ C) - -def test_contrast_from_column(D): - col = MS(['g']).fit(D).column_info_['g'] - assert contrast(col, 'drop').name == 'g' - -@pytest.mark.parametrize('drop_level', [2, '2']) -def test_contrast_int_drop_level(D, drop_level): - k = np.tile([1, 2, 3], D.shape[0] // 3 + 1)[:D.shape[0]] - for kk in [k, pd.Categorical(k)]: - X = MS([contrast('k', 'drop', drop_level)]).fit_transform(D.assign(k=kk)) - assert list(X.columns) == ['intercept', 'k[1]', 'k[3]'] - np.testing.assert_allclose(X.iloc[:, 1:], _indicators(k, [1, 3])) - -def test_contrast_ndarray_input(): - x = np.array(['b', 'a', 'c', 'a']) - C = Contrast(method=None).fit(x) - np.testing.assert_allclose(C.transform(x), _indicators(x, ['a', 'b', 'c'])) - np.testing.assert_allclose(C.transform(x.reshape((-1, 1))), _indicators(x, ['a', 'b', 'c'])) - -def test_contrast_unfit_methods_error(): - with pytest.raises(ValueError, match='method must be'): - Contrast(method='bogus').fit(np.array(['a', 'b'])) - -#### ordinal: opt-in - -_O_SCORE = {'lo': 0., 'mid': 1., 'hi': 2.} - -def test_ordered_categorical_default_is_contrast(D): - # ordered categoricals are coded like any other, in category order - X = MS(['o'], intercept=False).fit_transform(D) - assert list(X.columns) == ['o[mid]', 'o[hi]'] - np.testing.assert_allclose(X, _indicators(D.o, ['mid', 'hi'])) - _assert_same_design(X, MS([contrast('o')], intercept=False).fit_transform(D)) - -@pytest.mark.parametrize('how', ['helper', 'helper_column', 'column_key', 'ordinal_key']) -def test_ordinal_opt_in(D, how): - if how == 'helper': - M = MS([ordinal('o')], intercept=False) - elif how == 'helper_column': - M = MS([ordinal(MS(['o']).fit(D).column_info_['o'])], intercept=False) - elif how == 'column_key': - M = MS(['o'], categorical_encoders={'o': Ordinal()}, intercept=False) - else: - M = MS(['o', 'g'], categorical_encoders={'ordinal': Ordinal()}, intercept=False) - X = M.fit_transform(D) - # the helper names its column, an encoder keeps the column's name - name = 'ordinal(o)' if how.startswith('helper') else 'o' - np.testing.assert_allclose(X[name], D.o.map(_O_SCORE).astype(float)) - if how == 'ordinal_key': # only ordered columns use the 'ordinal' encoder - assert list(X.columns) == ['o', 'g[b]', 'g[c]'] - -@pytest.mark.parametrize('col', [ - pd.Series(['lo', 'mid', 'hi', 'lo'], dtype=object), # strings sort alphabetically - pd.Series(pd.Categorical(['lo', 'mid', 'hi', 'lo'])), # categories inferred, sorted - pd.Series(pd.Categorical(['lo', 'mid', 'hi', 'lo'], categories=['lo', 'mid', 'hi'])), -]) -def test_ordinal_ambiguous_order_raises(col): - # without levels=, only an ordered categorical (or numeric) has a trusted order - with pytest.raises(ValueError, match='ambiguous.*levels='): - MS([ordinal('o')]).fit(pd.DataFrame({'o': col})) - -@pytest.mark.parametrize('col', [ - pd.Series(['lo', 'mid', 'hi', 'lo'], dtype=object), - pd.Series(pd.Categorical(['lo', 'mid', 'hi', 'lo'])), -]) -def test_ordinal_levels(col): - X = MS([ordinal('o', levels=['lo', 'mid', 'hi'])], intercept=False).fit_transform(pd.DataFrame({'o': col})) - np.testing.assert_allclose(X['ordinal(o)'], [0, 1, 2, 0]) - -def test_ordinal_levels_errors(): - D = pd.DataFrame({'o': ['lo', 'mid', 'hi']}) - with pytest.raises(ValueError, match="unknown categories \\['mid'\\] during fit"): - MS([ordinal('o', levels=['lo', 'hi'])]).fit(D) - with pytest.raises(ValueError, match='duplicates'): - MS([ordinal('o', levels=['lo', 'lo', 'mid', 'hi'])]).fit(D) - -def test_ordinal_numeric_and_cut(): - D = pd.DataFrame({'k': [3, 1, 2, 3], 'inc': [10, 60, 120, 10]}) - D['band'] = pd.cut(D.inc, [0, 50, 90, 200], labels=['lo', 'mid', 'hi']) # ordered - X = MS([ordinal('k'), ordinal('band')], intercept=False).fit_transform(D) - np.testing.assert_allclose(X['ordinal(k)'], [2, 0, 1, 2]) - np.testing.assert_allclose(X['ordinal(band)'], [0, 1, 2, 0]) - -def test_ordinal_in_interaction(D): - X = MS([(ordinal('o'), 'x')], intercept=False).fit_transform(D) - np.testing.assert_allclose(X.iloc[:, 0], D.o.map(_O_SCORE).astype(float) * D.x) - -def test_ordinal_unused_level_keeps_positions(D): - Ds = D[D.o != 'mid'] # 'mid' unused, positions of lo/hi unchanged - M = MS([ordinal('o')], intercept=False).fit(Ds) - np.testing.assert_allclose(M.transform(D)['ordinal(o)'], D.o.map(_O_SCORE).astype(float)) - -def test_ordinal_nan_and_unknown(D): - o = D.o.astype(object) - o.iloc[:2] = np.nan - Dn = D.assign(o=pd.Categorical(o, categories=['lo', 'mid', 'hi'], ordered=True)) - X = MS([ordinal('o')], intercept=False).fit_transform(Dn) - assert X['ordinal(o)'].iloc[:2].isna().all() and not X['ordinal(o)'].iloc[2:].isna().any() - with pytest.raises(ValueError, match='unknown categories'): - MS([ordinal('o')]).fit(D).transform(D.assign(o=['top'] * D.shape[0])) - -def test_ordinal_user_encoder_respected(D): - # an explicitly supplied encoder is used as is - from sklearn.preprocessing import OrdinalEncoder - X = MS(['o'], categorical_encoders={'ordinal': OrdinalEncoder()}, intercept=False).fit_transform(D) - np.testing.assert_allclose(X['o'], D.o.map({'hi': 0., 'lo': 1., 'mid': 2.}).astype(float)) - -def test_wage_education_ordinal_vs_contrast(): - # Wage.education is ordered: contrasts by default, a linear score with ordinal() - W = load_data('Wage') - assert W.education.cat.ordered - X = MS(['education']).fit_transform(W) - assert X.shape[1] == 1 + W.education.cat.categories.shape[0] - 1 - Xo = MS([ordinal('education')]).fit_transform(W) - assert list(Xo.columns) == ['intercept', 'ordinal(education)'] - np.testing.assert_allclose(Xo['ordinal(education)'], W.education.cat.codes.astype(float)) - -@pytest.mark.parametrize('terms', [['s'], [('x', 's')], [poly('s', 2)]]) -@pytest.mark.parametrize('dtype', [object, 'str']) -def test_string_column_raises(D, terms, dtype): - # a string column must be made categorical explicitly - Ds = D.assign(s=D.g.astype(str).astype(dtype)) - with pytest.raises(ValueError, match="column 's' is not numeric.*astype\\('category'\\)"): - MS(terms).fit(Ds) - -def test_string_column_explicit_categorical(D): - Ds = D.assign(s=D.g.astype(str)) - expected = MS(['g']).fit_transform(D).values - np.testing.assert_allclose(MS([contrast('s')]).fit_transform(Ds), expected) - np.testing.assert_allclose(MS(['s']).fit_transform(Ds.assign(s=Ds.s.astype('category'))), expected) - -def test_unused_string_column_ok(D): - # columns not referenced by any term may be anything - X = MS(['x', 'g']).fit_transform(D.assign(label=D.g.astype(str))) - assert (X.dtypes == float).all() - -def test_object_column_of_numbers_ok(D): - X = MS(['x', 'b'], intercept=False).fit_transform(D.assign(x=D.x.astype(object), b=D.b.astype(object))) - np.testing.assert_allclose(X, np.column_stack([D.x, D.b]).astype(float)) - -def test_ndarray_string_column_raises(): - A = np.array([[1.0, 'a'], [2.0, 'b']], dtype=object) - with pytest.raises(ValueError, match="not numeric"): - MS([0, 1]).fit(A) - -# error messages - -@bug('missing column gives a generic "should be a Feature, Column..." message') -def test_missing_column_message(D): - with pytest.raises((KeyError, ValueError), match='nope'): - MS(['nope']).fit(D) - -@bug('BSpline/NaturalSpline with neither df nor knots: TypeError instead of ValueError') -@pytest.mark.parametrize('klass', [BSpline, NaturalSpline]) -def test_spline_requires_df_or_knots(klass): - with pytest.raises(ValueError): - klass().fit(np.linspace(0, 1, 10)) - -#### sklearn's estimator checks - -# `ModelSpec()` uses no columns, so these checks exercise input -# validation and the sklearn API rather than the model matrix. -_EXPECTED_FAILED_CHECKS = { - 'check_dtype_object': 'columns not used by any term are not validated', -} - -import sklearn -from sklearn.utils.fixes import parse_version -if parse_version(sklearn.__version__) >= parse_version('1.6'): - from sklearn.utils.estimator_checks import parametrize_with_checks - @parametrize_with_checks([MS()], - expected_failed_checks=lambda est: _EXPECTED_FAILED_CHECKS) - def test_sklearn_compatible(estimator, check): - check(estimator) - -#### other dataframe backends (via narwhals) - -def _carseats_like(D): - # D with an ordered categorical - return D.assign(band=pd.cut(D.x, [-1, 3, 7, 11], labels=['lo', 'mid', 'hi'])) - -_BACKEND_TERMS = ['x', 'g', ('x', 'g'), ns('z', df=3), poly('x', 2), 'b', ordinal('band')] - -def _to_polars(D): - pl = pytest.importorskip('polars') - # polars has no ordered Categorical: an Enum declares the order - return pl.from_pandas(D).with_columns( - pl.col('band').cast(pl.String).cast(pl.Enum(['lo', 'mid', 'hi'])), - pl.col('o').cast(pl.String).cast(pl.Enum(['lo', 'mid', 'hi']))) - -def _to_pyarrow(D): - pa = pytest.importorskip('pyarrow') - return pa.Table.from_pandas(D, preserve_index=False) - -def _as_pandas(X): - return X.to_pandas() if hasattr(X, 'to_pandas') else X - -@pytest.mark.parametrize('convert, module', [(_to_polars, 'polars'), (_to_pyarrow, 'pyarrow')]) -def test_backend_matches_pandas(D, convert, module): - D = _carseats_like(D) - N = convert(D) - expected = MS(_BACKEND_TERMS).fit_transform(D) - for X in [MS(_BACKEND_TERMS).fit_transform(N), # fit and transform - MS(_BACKEND_TERMS).fit(D).transform(N)]: # fit pandas, transform other - assert type(X).__module__.split('.')[0] == module - _assert_same_design(_as_pandas(X), expected.reset_index(drop=True)) - # fit other, transform pandas - _assert_same_design(MS(_BACKEND_TERMS).fit(N).transform(D), expected) - -@pytest.mark.parametrize('convert', [_to_polars, _to_pyarrow]) -def test_backend_submodel_sequence_names(D, convert): - D = _carseats_like(D) - N = convert(D) - M = MS(_BACKEND_TERMS).fit(N) - assert list(M.get_feature_names_out()) == list(_as_pandas(M.transform(N)).columns) - assert list(M.feature_names_in_) == list(D.columns) - ref = MS(_BACKEND_TERMS).fit(D) - _assert_same_design(_as_pandas(M.build_submodel(N, _BACKEND_TERMS[:2])), - ref.build_submodel(D, _BACKEND_TERMS[:2]).reset_index(drop=True)) - for A, B in zip(M.build_sequence(N), ref.build_sequence(D)): - _assert_same_design(_as_pandas(A), B.reset_index(drop=True)) - -def test_polars_categorical_levels_sorted(): - # a polars Categorical stores levels in order of appearance: sorted, - # as pandas does, so the baseline does not depend on row order - pl = pytest.importorskip('polars') - for values in (['b', 'a', 'c', 'a'], ['c', 'b', 'a', 'a']): - P = pl.DataFrame({'g': pl.Series(values, dtype=pl.Categorical)}) - assert list(MS(['g'], intercept=False).fit_transform(P).columns) == ['g[b]', 'g[c]'] - -def test_polars_enum_and_categorical_ordinal(): - pl = pytest.importorskip('polars') - E = pl.DataFrame({'e': pl.Series(['hi', 'lo', 'mid'], dtype=pl.Enum(['lo', 'mid', 'hi']))}) - np.testing.assert_allclose(MS([ordinal('e')], intercept=False).fit_transform(E)['ordinal(e)'].to_numpy(), [2, 0, 1]) - P = pl.DataFrame({'g': pl.Series(['b', 'a'], dtype=pl.Categorical)}) - with pytest.raises(ValueError, match='ambiguous'): - MS([ordinal('g')]).fit(P) - -def test_polars_series_string_lazy_duplicates(D): - pl = pytest.importorskip('polars') - P = pl.from_pandas(D[['x', 'z']]) - X = MS(['x']).fit_transform(P['x']) - assert isinstance(X, pl.DataFrame) and X.columns == ['intercept', 'x'] - with pytest.raises(ValueError, match="column 's' is not numeric"): - MS(['s']).fit(pl.DataFrame({'s': ['u', 'v']})) - with pytest.raises(TypeError, match='Lazy frames'): - MS(['x']).fit(P.lazy()) - with pytest.raises(ValueError, match='duplicated column names'): - MS(['x', 'x']).fit_transform(P) - -def test_polars_set_output_and_pipeline(D): - pl = pytest.importorskip('polars') - D = _carseats_like(D) - P = _to_polars(D) - assert isinstance(MS(_BACKEND_TERMS).set_output(transform='pandas').fit_transform(P), pd.DataFrame) - from sklearn.utils.fixes import parse_version - if parse_version(sklearn.__version__) >= parse_version('1.4'): # polars output - assert isinstance(MS(_BACKEND_TERMS).set_output(transform='polars').fit_transform(D), pl.DataFrame) - pipe = make_pipeline(MS(_BACKEND_TERMS), LinearRegression()) - np.testing.assert_allclose(cross_val_score(pipe, P, P['z'], cv=4, error_score='raise'), - cross_val_score(pipe, D, D['z'], cv=4, error_score='raise')) - -def test_build_sequence_bool_column(D): - # boolean columns are float in build_sequence, as in transform, - # so that an ANOVA over the sequence can be fit - from statsmodels.stats.anova import anova_lm - M = MS(['x', 'b']).fit(D) - seq = list(M.build_sequence(D)) - assert all((X.dtypes == float).all() for X in seq) - _assert_same_design(seq[-1], M.transform(D)) - anova_lm(*[sm.OLS(D.z, X).fit() for X in seq]) - -#### sparse_output - -import scipy.sparse - -def _sparse_terms(): - return ['x', 'g', ('x', 'g'), ('g', 'o'), contrast('o', 'sum'), contrast('g', None), - poly('z', 2), ns('z', df=3), 'b', ordinal('o'), ('g', 'z'), (poly('z', 2), 'g'), - pca(['x', 'z'], 'xz', n_components=1)] - -def _with_missing(D): - g = D.g.astype(object) - g.iloc[:3] = np.nan - return D.assign(gn=pd.Categorical(g)) - -def test_sparse_matches_dense(D): - D = _with_missing(D) - terms = _sparse_terms() + ['gn', ('gn', 'x')] - M = MS(terms, sparse_output=True).fit(D) - X = M.transform(D) - assert isinstance(X, scipy.sparse.csr_matrix) - dense = MS(terms).fit_transform(D) - np.testing.assert_allclose(X.toarray(), dense, equal_nan=True) - assert list(M.get_feature_names_out()) == list(dense.columns) - -def test_sparse_submodel_sequence(D): - terms = _sparse_terms() - M, Md = MS(terms, sparse_output=True).fit(D), MS(terms).fit(D) - np.testing.assert_allclose(M.build_submodel(D, terms[:3]).toarray(), Md.build_submodel(D, terms[:3])) - for how in ['sequential', 'drop']: - for A, B in zip(M.build_sequence(D, how), Md.build_sequence(D, how)): - assert scipy.sparse.issparse(A) - np.testing.assert_allclose(A.toarray(), B) - M0 = MS([], intercept=False, sparse_output=True).fit(D) - assert M0.transform(D).shape == (D.shape[0], 1) - -def test_sparse_ndarray_and_polars(D): - A = D[['x', 'z']].values - np.testing.assert_allclose(MS([0, (0, 1)], sparse_output=True).fit_transform(A).toarray(), - MS([0, (0, 1)]).fit_transform(A)) - pl = pytest.importorskip('polars') - P = pl.from_pandas(D[['x', 'g']]) - X = MS(['x', 'g', ('x', 'g')], sparse_output=True).fit_transform(P) - assert scipy.sparse.issparse(X) # not converted to a polars frame - np.testing.assert_allclose(X.toarray(), MS(['x', 'g', ('x', 'g')]).fit_transform(D)) - -def test_sparse_many_levels_memory(): - # a categorical with many levels, and its interaction, are never dense: - # the dense design would be n * K * 8 = 2 GB - import tracemalloc - rng = np.random.default_rng(0) - n, K = 50_000, 5_000 - B = pd.DataFrame({'x': rng.standard_normal(n), - 'id': pd.Categorical(rng.integers(0, K, n))}) - for terms in (['id', 'x'], [('id', 'x')]): - tracemalloc.start() - X = MS(terms, sparse_output=True).fit_transform(B) - peak = tracemalloc.get_traced_memory()[1] - tracemalloc.stop() - assert X.shape[1] >= K - 1 and X.nnz <= 3 * n - assert peak < 100e6, peak - -def test_sparse_encoder_output(D): - # an encoder returning sparse output: kept in sparse mode, - # densified otherwise (np.asarray of a sparse matrix is a 0-d object array) - from sklearn.preprocessing import OneHotEncoder - enc = {'g': OneHotEncoder()} - Xd = MS(['g'], categorical_encoders=enc, intercept=False).fit_transform(D) - np.testing.assert_allclose(Xd, _indicators(D.g, ['a', 'b', 'c'])) - Xs = MS(['g'], categorical_encoders=enc, intercept=False, sparse_output=True).fit_transform(D) - assert scipy.sparse.issparse(Xs) - np.testing.assert_allclose(Xs.toarray(), Xd) - -def test_sparse_pipeline(D): - # same design, so the same fit (LinearRegression's solver is exact); - # terms with distinct column names, which sklearn >= 1.9 requires of - # a pd.DataFrame (contrast('g', None) repeats 'g''s names) - terms = _sparse_terms()[:5] + _sparse_terms()[6:7] - sparse = cross_val_score(make_pipeline(MS(terms, sparse_output=True), LinearRegression()), - D, D.z, cv=4, error_score='raise') - dense = cross_val_score(make_pipeline(MS(terms), LinearRegression()), - D, D.z, cv=4, error_score='raise') - np.testing.assert_allclose(sparse, dense) - -def test_sparse_errors(D): - with pytest.raises(ValueError, match='sparse_output should be True or False'): - MS(['g'], sparse_output='yes').fit(D) - with pytest.raises(ValueError, match='sparse'): # sklearn: pandas output cannot be sparse - MS(['g'], sparse_output=True).set_output(transform='pandas').fit_transform(D) - -def test_contrast_matrix_still_dense(D): - C = Contrast(method='sum').fit(D.g) - assert isinstance(C.contrast_matrix_, np.ndarray) - np.testing.assert_allclose(C.contrast_matrix_, [[1, 0], [0, 1], [-1, -1]]) - -#### memory: fit does not build the model matrix, transform builds it once - -def _traced_peak(f): - import tracemalloc - tracemalloc.start() - try: - result = f() - return result, tracemalloc.get_traced_memory()[1] - finally: - tracemalloc.stop() - -def _big_design_data(n=200_000, ncol=2): - rng = np.random.default_rng(0) - data = {f'x{j}': rng.standard_normal(n) for j in range(ncol)} - data['g'] = pd.Categorical(rng.integers(0, 50, n).astype(str)) - return pd.DataFrame(data) - -@pytest.mark.parametrize('terms', [['x0', 'x1', 'g'], - ['x0', ns('x1', df=5), poly('x0', 3), 'g']]) -def test_fit_does_not_build_model_matrix(terms): - B = _big_design_data() - M = MS(terms, intercept=True) - _, fit_peak = _traced_peak(lambda: M.fit(B)) - X, transform_peak = _traced_peak(lambda: M.transform(B)) - size = X.values.nbytes - # fitting ns and poly needs their inputs (and a QR decomposition), - # but not the model matrix - assert fit_peak < 0.5 * size, (fit_peak, size) - # the model matrix and (at most) its largest block - assert transform_peak < 2.5 * size, (transform_peak, size) - -def test_dense_output_is_float(D): - X = MS(['x', 'b', 'g', ('x', 'g')], intercept=True).fit_transform(D.assign(x=np.arange(D.shape[0]))) - assert (X.dtypes == float).all() - assert isinstance(X, pd.DataFrame) and (X.index == D.index).all() - -def test_polars_converts_only_used_columns(): - # only the columns the terms use are converted to pandas (polars - # allocates outside Python, so this is checked directly, not by memory) - pl = pytest.importorskip('polars') - wide = _big_design_data(n=1_000, ncol=30) - P = pl.from_pandas(wide) - M = MS(['x0', ('x1', 'g')], intercept=True).fit(P) - assert set(M.column_info_) == {'x0', 'x1', 'g'} - # but the spec describes all of its input, as for pandas - assert M.n_features_in_ == wide.shape[1] - assert list(M.feature_names_in_) == list(wide.columns) - _assert_same_design(_as_pandas(M.transform(P)), - MS(['x0', ('x1', 'g')], intercept=True).fit_transform(wide).reset_index(drop=True)) - -#### parameters of the terms: set_params / GridSearchCV over e.g. terms__0__df - -def test_term_params_listed(): - from sklearn.preprocessing import SplineTransformer - sp = derived_feature(['x'], encoder=SplineTransformer(n_knots=5), name='spl') - M = MS([ns('x', df=4), 'g', (poly('z', 2), 'g'), contrast('g', 'sum'), sp], intercept=False) - deep = M.get_params(deep=True) - assert deep['terms__0__df'] == 4 - assert deep['terms__2__0__degree'] == 2 - assert deep['terms__3__method'] == 'sum' - assert deep['terms__4__n_knots'] == 5 - assert not any(k.startswith('terms__1__') for k in deep) # a plain column has none - assert not any(k.startswith('terms__') for k in M.get_params(deep=False)) - -@pytest.mark.parametrize('term, param, value, name', [ - (ns('x', df=4), 'df', 6, 'ns(x, df=6)'), - (bs('x', df=4), 'df', 7, 'bs(x, df=7)'), - (poly('x', 2), 'degree', 3, 'poly(x, degree=3)'), - (pca(['x', 'z'], 'xz', n_components=1), 'n_components', 2, 'pca(xz, n_components=2)'), -]) -def test_set_term_param_rebuilds(D, term, param, value, name): - terms = [term, 'g'] - M = MS(terms, intercept=False).set_params(**{f'terms__0__{param}': value}) - assert M.terms[0].name == name - assert terms[0] is term and M.terms is not terms # terms itself is not modified - expected = {'ns': ns, 'bs': bs, 'poly': poly} - if name.startswith('pca'): - ref = pca(['x', 'z'], 'xz', n_components=2) - else: - ref = expected[name[:name.index('(')]]('x', **{param: value}) - _assert_same_design(M.fit_transform(D), MS([ref, 'g'], intercept=False).fit_transform(D)) - -def test_set_term_param_named_and_interaction(D): - M = MS([ns('x', df=4, name='xs'), ('g', poly('z', 2))], intercept=False) - M.set_params(terms__xs__df=6, terms__1__1__degree=3) - X = M.fit_transform(D) - assert [c for c in X.columns if c.startswith('xs')] == [f'xs[{i}]' for i in range(6)] - assert M.terms[1][1].name == 'poly(z, degree=3)' - _assert_same_design(X, MS([ns('x', df=6, name='xs'), ('g', poly('z', 3))], intercept=False).fit_transform(D)) - -def test_set_term_param_shared_term(D): - # a term used in several places is changed in all of them - p = poly('z', 2) - M = MS([p, (p, 'g')], intercept=False).set_params(terms__0__degree=3) - assert M.terms[0].name == 'poly(z, degree=3)' and M.terms[1][0] is M.terms[0] - assert M.fit_transform(D).shape[1] == 3 + 3 * 2 - -def test_set_term_param_encoders(D): - from sklearn.preprocessing import SplineTransformer - sp = derived_feature(['x'], encoder=SplineTransformer(n_knots=5), name='spl') - M = MS([sp, contrast('g', 'drop'), ordinal('o')], intercept=False) - M.set_params(terms__0__n_knots=7, terms__1__method=None, terms__2__levels=['hi', 'mid', 'lo']) - assert sp.encoder.n_knots == 5 # the original is not changed - X = M.fit_transform(D) - assert sum(c.startswith('spl') for c in X.columns) == 7 + 3 - 1 - assert [c for c in X.columns if c.startswith('g[')] == ['g[a]', 'g[b]', 'g[c]'] - np.testing.assert_allclose(X['ordinal(o)'], D.o.map({'hi': 0., 'mid': 1., 'lo': 2.}).astype(float)) - P = MS([pca(['x', 'z'], 'xz', n_components=1)], intercept=False).set_params(terms__0__scale=True) - _assert_same_design(P.fit_transform(D), - MS([pca(['x', 'z'], 'xz', scale=True, n_components=1)], intercept=False).fit_transform(D)) - -def test_gridsearch_over_term_params(D): - pipe = make_pipeline(MS([ns('x', df=4), 'g'], intercept=False), LinearRegression()) - nested = GridSearchCV(pipe, {'modelspec__terms__0__df': [1, 2, 4]}, cv=3).fit(D, D.z) - explicit = GridSearchCV(make_pipeline(MS(intercept=False), LinearRegression()), - {'modelspec__terms': [[ns('x', df=d), 'g'] for d in (1, 2, 4)]}, - cv=3).fit(D, D.z) - np.testing.assert_allclose(nested.cv_results_['mean_test_score'], - explicit.cv_results_['mean_test_score']) - best = nested.best_params_['modelspec__terms__0__df'] - assert nested.best_estimator_[0].terms[0].name == f'ns(x, df={best})' - -def test_set_term_param_clone_pickle(D): - M = MS([ns('x', df=4), ('g', poly('z', 2))], intercept=False).set_params(terms__0__df=5) - C = clone(M) - assert C.terms[0].name == 'ns(x, df=5)' - P = pickle.loads(pickle.dumps(M.fit(D))) - _assert_same_design(P.transform(D), M.transform(D)) - -def test_set_params_terms_then_term_params(D): - M = MS(['x'], intercept=False).set_params(terms=[ns('z', df=3)], terms__0__df=5) - assert M.terms[0].name == 'ns(z, df=5)' - -@pytest.mark.parametrize('key, match', [ - ('terms__5__df', 'no term at position 5'), - ('terms__nope__df', "name of exactly one term, got 'nope'"), - ('terms__0__bogus', "invalid parameter 'bogus'.*valid parameters"), - ('terms__1__df', "the term 'g' has no parameters"), - ('terms__2__df', 'position of a part of the interaction'), -]) -def test_set_term_param_errors(key, match): - M = MS([ns('x', df=4), 'g', ('g', poly('z', 2))], intercept=False) - with pytest.raises(ValueError, match=match): - M.set_params(**{key: 1}) diff --git a/tests/models/test_modelmatrix.py b/tests/models/test_modelmatrix.py new file mode 100644 index 0000000..6ae0a27 --- /dev/null +++ b/tests/models/test_modelmatrix.py @@ -0,0 +1,93 @@ +# ModelSpec is modelmatrix's: its imports from ISLP, and model selection +# with its terms (ModelSpec itself is tested in modelmatrix) + +import numpy as np, pandas as pd +import pytest +import statsmodels.api as sm + +import modelmatrix +from ISLP.models import (ModelSpec as MS, + Stepwise, + sklearn_selected, + poly, + wildcard) + +bug = lambda reason: pytest.mark.xfail(strict=True, reason=reason) + + +@pytest.fixture +def D(): + rng = np.random.default_rng(0) + n = 60 + D = pd.DataFrame({'x': rng.uniform(0, 10, n), + 'z': rng.standard_normal(n), + 'u': rng.standard_normal(n), + 'g': pd.Categorical(rng.choice(['a', 'b', 'c'], n))}) + D['y'] = D['x'] + 2 * D['z'] + rng.standard_normal(n) + return D + + +def test_imports(): + import ISLP.models + from ISLP.models.model_spec import ModelSpec, Contrast, build_model + from ISLP.models.columns import Column + from ISLP.transforms import Poly, NaturalSpline + for name in modelmatrix.__all__: + if hasattr(ISLP.models, name): + assert getattr(ISLP.models, name) is getattr(modelmatrix, name) + assert ModelSpec is modelmatrix.ModelSpec + assert Contrast is modelmatrix.Contrast + assert Column is modelmatrix.Column + assert Poly is modelmatrix.transforms.Poly + assert callable(build_model) and NaturalSpline is modelmatrix.transforms.NaturalSpline + + +def test_intercept_default_is_False(D): + assert MS(['x']).get_params()['intercept'] is False + assert list(MS(['x']).fit_transform(D).columns) == ['x'] + + +def _selected(model_spec, D, **kwargs): + strategy = Stepwise.first_peak(model_spec, direction='forward', + max_terms=len(model_spec.terms_), **kwargs) + return sklearn_selected(sm.OLS, strategy).fit(D, D['y']).selected_state_ + + +def test_stepwise_with_wildcard(D): + # the wildcard stands for several terms: the same selection as with + # the columns given one by one + explicit = MS(['x', 'z', 'u', 'g'], intercept=True).fit(D) + dot = MS([wildcard(exclude='y')], intercept=True).fit(D) + assert len(dot.terms) == 1 and len(dot.terms_) == 4 + assert _selected(dot, D) == _selected(explicit, D) + assert set(_selected(dot, D)) >= {'x', 'z'} + + +def test_stepwise_lower_terms_among_the_wildcard(D): + # a column that the wildcard stands for can be a lower term + dot = MS([wildcard(exclude='y')], intercept=True).fit(D) + assert 'u' in _selected(dot, D, lower_terms=['u']) + + +def test_fixed_steps_backward_with_wildcard(D): + dot = MS([wildcard(exclude='y')], intercept=True).fit(D) + strategy = Stepwise.fixed_steps(dot, 1, direction='backward') + selected = sklearn_selected(sm.OLS, strategy).fit(D, D['y']).selected_state_ + assert len(selected) == len(dot.terms_) - 1 + + +@bug('_postprocess_best reduces state to names; build_submodel cannot resolve derived names') +@pytest.mark.parametrize('term', [poly('z', 2), ('x', 'z')]) +def test_stepwise_with_derived_term(D, term): + M = MS(['x', term, 'g'], intercept=True).fit(D) + strategy = Stepwise.first_peak(M, direction='forward', max_terms=3) + sklearn_selected(sm.OLS, strategy).fit(D, D.x + D.z) + + +def test_best_1sd_picks_smaller_model(): + from ISLP.models.strategy import _postprocess_best_1sd + class T(str): + name = property(str) + results = [((T('a'),), 0, np.array([-1.0, -1.1, -0.9])), + ((T('a'), T('b')), 1, np.array([-0.98, -0.99, -0.97]))] + assert _postprocess_best_1sd(results)[0] == ('a',) diff --git a/tests/models/test_selection.py b/tests/models/test_selection.py index ba13d17..dfa737c 100644 --- a/tests/models/test_selection.py +++ b/tests/models/test_selection.py @@ -6,7 +6,7 @@ from sklearn.linear_model import LinearRegression from ISLP.models import ModelSpec as MS -from ISLP.models.model_spec import Feature +from ISLP.models import Feature from ISLP.models.strategy import (min_max, Stepwise, validator_from_constraints, diff --git a/tests/models/test_sklearn_wrap.py b/tests/models/test_sklearn_wrap.py index 4fca67f..3f974e4 100644 --- a/tests/models/test_sklearn_wrap.py +++ b/tests/models/test_sklearn_wrap.py @@ -6,7 +6,7 @@ import pytest from ISLP.models.sklearn_wrap import sklearn_sm, sklearn_selected -from ISLP.models.model_spec import ModelSpec +from ISLP.models import ModelSpec from ISLP.models.strategy import min_max @pytest.fixture