Feature importance methods for black box models

Expand source code 
"""
Feature importance methods for black box models
"""

from .rf_plus import RandomForestPlusRegressor, RandomForestPlusClassifier
from .mdi_plus import ForestMDIPlus, TreeMDIPlus
from .ppms import GenericRegressorPPM, GenericClassifierPPM, \
    GlmRegressorPPM, GlmClassifierPPM, RidgeRegressorPPM, RidgeClassifierPPM, \
    LogisticClassifierPPM, RobustRegressorPPM, LassoRegressorPPM
from .block_transformers import IdentityTransformer, TreeTransformer, \
    CompositeTransformer, MDIPlusDefaultTransformer

# re-exported for callers; listed so the intent is explicit
__all__ = [
    "CompositeTransformer", "ForestMDIPlus", "GenericClassifierPPM",
    "GenericRegressorPPM", "GlmClassifierPPM", "GlmRegressorPPM",
    "IdentityTransformer", "LassoRegressorPPM", "LogisticClassifierPPM",
    "MDIPlusDefaultTransformer", "RandomForestPlusClassifier",
    "RandomForestPlusRegressor", "RidgeClassifierPPM", "RidgeRegressorPPM",
    "RobustRegressorPPM", "TreeMDIPlus", "TreeTransformer",
]

Sub-modules

imodels.importance.block_transformers
imodels.importance.block_transformers_ys
imodels.importance.local_stumps
imodels.importance.mdi_plus
imodels.importance.ppms
imodels.importance.ranking_stability
imodels.importance.rf_plus

Classes

class CompositeTransformer (block_transformer_list, rescale_mode=None, drop_features=True)

A block transformer that is built by concatenating the blocks of the same index from a list of block transformers.

Parameters

block_transformer_list : list of BlockTransformer objects
The list of block transformers to combine
rescale_mode : string in {"max", "mean", None}
Flag for the type of rescaling to be done to the blocks from different base transformers. If "max", divide each block by the max std deviation of a column within the block. If "mean", divide each block by the mean std deviation of a column within the block. If None, do not rescale.
drop_features : bool
Flag for whether to return an empty block if that from the first transformer in the list is trivial.
Expand source code 
class CompositeTransformer(BlockTransformerBase, ABC):
    """
    A block transformer that is built by concatenating the blocks of the same
    index from a list of block transformers.

    Parameters
    ----------
    block_transformer_list: list of BlockTransformer objects
        The list of block transformers to combine
    rescale_mode: string in {"max", "mean", None}
        Flag for the type of rescaling to be done to the blocks from different
        base transformers. If "max", divide each block by the max std deviation
        of a column within the block. If "mean", divide each block by the mean
        std deviation of a column within the block. If None, do not rescale.
    drop_features: bool
        Flag for whether to return an empty block if that from the first
        transformer in the list is trivial.
    """

    def __init__(self, block_transformer_list, rescale_mode=None, drop_features=True):
        super().__init__()
        self.block_transformer_list = block_transformer_list
        assert len(self.block_transformer_list) > 0, "Need at least one base" \
                                                     "transformer."
        for transformer in block_transformer_list:
            if hasattr(transformer, "oob_seed") and \
                    transformer.oob_seed is not None:
                self.oob_seed = transformer.oob_seed
                break
        self.rescale_mode = rescale_mode
        self.drop_features = drop_features
        self._rescale_factors = {}
        self._trivial_block_indices = {}

    def _fit_one_feature(self, X, k):
        data_blocks = []
        for block_transformer in self.block_transformer_list:
            data_block = block_transformer.fit_transform_one_feature(
                X, k, center=False, normalize=False)
            data_blocks.append(data_block)

        # Handle trivial blocks
        self._trivial_block_indices[k] = \
            [idx for idx, data_block in enumerate(data_blocks) if
             _empty_or_constant(data_block)]
        if (0 in self._trivial_block_indices[k] and self.drop_features) or \
                (len(self._trivial_block_indices[k]) == len(data_blocks)):
            # If first block is trivial and self.drop_features is True,
            self._centers[k] = np.array([0])
            self._scales[k] = np.array([1])
            return
        else:
            # Remove trivial blocks
            for idx in reversed(self._trivial_block_indices[k]):
                data_blocks.pop(idx)
        self._rescale_factors[k] = _get_rescale_factors(data_blocks, self.rescale_mode)
        composite_block = np.hstack(
            [data_block / scale_factor for data_block, scale_factor in
             zip(data_blocks, self._rescale_factors[k])]
        )
        self._centers[k] = composite_block.mean(axis=0)
        self._scales[k] = composite_block.std(axis=0)

    def _transform_one_feature(self, X, k):
        data_blocks = []
        for block_transformer in self.block_transformer_list:
            data_block = block_transformer.transform_one_feature(
                X, k, center=False, normalize=False)
            data_blocks.append(data_block)
        # Handle trivial blocks
        if (0 in self._trivial_block_indices[k] and self.drop_features) or \
                (len(self._trivial_block_indices[k]) == len(data_blocks)):
            # If first block is trivial and self.drop_features is True,
            # return empty block
            return np.empty((X.shape[0], 0))
        else:
            # Remove trivial blocks
            for idx in reversed(self._trivial_block_indices[k]):
                data_blocks.pop(idx)
        composite_block = np.hstack(
            [data_block / scale_factor for data_block, scale_factor in
             zip(data_blocks, self._rescale_factors[k])]
        )
        return composite_block

    def _fit_transform_one_feature(self, X, k):
        data_blocks = []
        for block_transformer in self.block_transformer_list:
            data_block = block_transformer.fit_transform_one_feature(
                X, k, center=False, normalize=False)
            data_blocks.append(data_block)
        # Handle trivial blocks
        self._trivial_block_indices[k] = \
            [idx for idx, data_block in enumerate(data_blocks) if
             _empty_or_constant(data_block)]
        if (0 in self._trivial_block_indices[k] and self.drop_features) or \
                (len(self._trivial_block_indices[k]) == len(data_blocks)):
            # If first block is trivial and self.drop_features is True,
            # return empty block
            self._centers[k] = np.array([0])
            self._scales[k] = np.array([1])
            return np.empty((X.shape[0], 0))
        else:
            # Remove trivial blocks
            for idx in reversed(self._trivial_block_indices[k]):
                data_blocks.pop(idx)
        self._rescale_factors[k] = _get_rescale_factors(data_blocks, self.rescale_mode)
        composite_block = np.hstack(
            [data_block / scale_factor for data_block, scale_factor in
             zip(data_blocks, self._rescale_factors[k])]
        )
        self._centers[k] = composite_block.mean(axis=0)
        self._scales[k] = composite_block.std(axis=0)
        return composite_block

Ancestors

Subclasses

Inherited members

class ForestMDIPlus (estimators, transformers, scoring_fns, sample_split='loo', tree_random_states=None, mode='keep_k', task='regression', center=True, normalize=False)

The class object for computing MDI+ feature importances for a forest or collection of trees. Generalized mean decrease in impurity (MDI+) is a flexible framework for computing RF feature importances. For more details, refer to [paper].

Parameters

estimators : list of fitted PartialPredictionModelBase objects or scikit-learn type estimators
The fitted partial prediction models (one per tree) to use for evaluating feature importance via MDI+. If not a PartialPredictionModelBase, then the estimator is coerced into a PartialPredictionModelBase object via GenericRegressorPPM or GenericClassifierPPM depending on the specified task. Note that these generic PPMs may be computationally expensive.
transformers : list of BlockTransformerBase objects
The block feature transformers used to generate blocks of engineered features for each original feature. The transformed data is then used as input into the partial prediction models. Should be the same length as estimators.
scoring_fns : a function or dict with functions as value and function name (str) as key
The scoring functions used for evaluating the partial predictions.
sample_split : string in {"loo", "oob", "inbag"} or None
The sample splitting strategy to be used when evaluating the partial model predictions. The default "loo" (leave-one-out) is strongly recommended for performance and in particular, for overcoming the known correlation and entropy biases suffered by MDI. "oob" (out-of-bag) can also be used to overcome these biases. "inbag" is the sample splitting strategy used by MDI. If None, no sample splitting is performed and the full data set is used to evaluate the partial model predictions.
tree_random_states : list of int or None
Random states from each tree in the fitted random forest; used in sample splitting and only required if sample_split = "oob" or "inbag". Should be the same length as estimators.
mode : string in {"keep_k", "keep_rest"}
Mode for the method. "keep_k" imputes the mean of each feature not in block k when making a partial model prediction, while "keep_rest" imputes the mean of each feature in block k. "keep_k" is strongly recommended for computational considerations.
task : string in {"regression", "classification"}
The supervised learning task for the RF model. Used for choosing defaults for the scoring_fns. Currently only regression and classification are supported.
center : bool
Flag for whether to center the transformed data in the transformers.
normalize : bool
Flag for whether to rescale the transformed data to have unit variance in the transformers.
Expand source code 
class ForestMDIPlus:
    """
    The class object for computing MDI+ feature importances for a forest or collection of trees.
    Generalized mean decrease in impurity (MDI+) is a flexible framework for computing RF
    feature importances. For more details, refer to [paper].

    Parameters
    ----------
    estimators: list of fitted PartialPredictionModelBase objects or scikit-learn type estimators
        The fitted partial prediction models (one per tree) to use for evaluating
        feature importance via MDI+. If not a PartialPredictionModelBase, then
        the estimator is coerced into a PartialPredictionModelBase object via
        GenericRegressorPPM or GenericClassifierPPM depending on the specified
        task. Note that these generic PPMs may be computationally expensive.
    transformers: list of BlockTransformerBase objects
        The block feature transformers used to generate blocks of engineered
        features for each original feature. The transformed data is then used
        as input into the partial prediction models. Should be the same length
        as estimators.
    scoring_fns: a function or dict with functions as value and function name (str) as key
        The scoring functions used for evaluating the partial predictions.
    sample_split: string in {"loo", "oob", "inbag"} or None
        The sample splitting strategy to be used when evaluating the partial
        model predictions. The default "loo" (leave-one-out) is strongly
        recommended for performance and in particular, for overcoming the known
        correlation and entropy biases suffered by MDI. "oob" (out-of-bag) can
        also be used to overcome these biases. "inbag" is the sample splitting
        strategy used by MDI. If None, no sample splitting is performed and the
        full data set is used to evaluate the partial model predictions.
    tree_random_states: list of int or None
        Random states from each tree in the fitted random forest; used in
        sample splitting and only required if sample_split = "oob" or "inbag".
        Should be the same length as estimators.
    mode: string in {"keep_k", "keep_rest"}
        Mode for the method. "keep_k" imputes the mean of each feature not
        in block k when making a partial model prediction, while "keep_rest"
        imputes the mean of each feature in block k. "keep_k" is strongly
        recommended for computational considerations.
    task: string in {"regression", "classification"}
        The supervised learning task for the RF model. Used for choosing
        defaults for the scoring_fns. Currently only regression and
        classification are supported.
    center: bool
        Flag for whether to center the transformed data in the transformers.
    normalize: bool
        Flag for whether to rescale the transformed data to have unit
        variance in the transformers.
    """

    def __init__(self, estimators, transformers, scoring_fns,
                 sample_split="loo", tree_random_states=None, mode="keep_k",
                 task="regression", center=True, normalize=False):
        assert sample_split in ["loo", "oob", "inbag", None]
        assert mode in ["keep_k", "keep_rest"]
        assert task in ["regression", "classification"]
        self.estimators = estimators
        self.transformers = transformers
        self.scoring_fns = scoring_fns
        self.sample_split = sample_split
        self.tree_random_states = tree_random_states
        if self.sample_split in ["oob", "inbag"] and not self.tree_random_states:
            raise ValueError("Must specify tree_random_states to use 'oob' or 'inbag' sample_split.")
        self.mode = mode
        self.task = task
        self.center = center
        self.normalize = normalize
        self.is_fitted = False
        self.prediction_score_ = pd.DataFrame({})
        self.feature_importances_ = pd.DataFrame({})
        self.feature_importances_by_tree_ = {}

    def get_scores(self, X, y):
        """
        Obtain the MDI+ feature importances for a forest.

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            The covariate matrix. If a pd.DataFrame object is supplied, then
            the column names are used in the output
        y: ndarray of shape (n_samples, n_targets)
            The observed responses.

        Returns
        -------
        scores: pd.DataFrame of shape (n_features, n_scoring_fns)
            The MDI+ feature importances.
        """
        self._fit_importance_scores(X, y)
        return self.feature_importances_

    def get_stability_scores(self, B=10, metrics="auto"):
        """
        Evaluate the stability of the MDI+ feature importance rankings
        across bootstrapped samples of trees. Can be used to select the GLM
        and scoring metric in a data-driven manner, where the GLM and metric that
        yields the most stable feature rankings across bootstrapped samples is selected.

        Parameters
        ----------
        B: int
            Number of bootstrap samples.
        metrics: "auto" or a dict with functions as value and function name (str) as key
            Metric(s) used to evaluate the stability between two sets of feature importances.
            If "auto", then the feature importance stability metrics are:
                (1) Rank-based overlap (RBO) with p=0.9 (from "A Similarity Measure for
                Indefinite Rankings" by Webber et al. (2010)). Intuitively, this metric gives
                more weight to features with the largest importances, with most of the weight
                going to the ~1/(1-p) features with the largest importances.
                (2) A weighted kendall tau metric (tauAP_b from "The Treatment of Ties in
                AP Correlation" by Urbano and Marrero (2017)), which also gives more weight
                to the features with the largest importances, but uses a different weighting
                scheme from RBO.
            Note that these default metrics assume that a higher MDI+ score indicates
            greater importance and thus give more weight to these features with high
            importance/ranks. If a lower MDI+ score indicates higher importance, then invert
            either these stability metrics or the MDI+ scores before evaluating the stability.

        Returns
        -------
        stability_results: pd.DataFrame of shape (n_features, n_metrics)
            The stability scores of the MDI+ feature rankings across bootstrapped samples.

        """
        if metrics == "auto":
            metrics = {"RBO": partial(rbo, p=0.9), "tauAP": tauAP_b}
        elif not isinstance(metrics, dict):
            raise ValueError("`metrics` must be 'auto' or a dictionary "
                             "where the key is the metric name and the value is the evaluation function")
        single_scoring_fn = not isinstance(self.feature_importances_by_tree_, dict)
        if single_scoring_fn:
            feature_importances_dict = {"mdi_plus_score": self.feature_importances_by_tree_}
        else:
            feature_importances_dict = self.feature_importances_by_tree_
        stability_dict = {}
        for scoring_fn_name, feature_importances_by_tree in feature_importances_dict.items():
            n_trees = feature_importances_by_tree.shape[1]
            fi_scores_boot_ls = []
            for b in range(B):
                bootstrap_sample = np.random.choice(n_trees, n_trees, replace=True)
                fi_scores_boot_ls.append(feature_importances_by_tree[bootstrap_sample].mean(axis=1))
            fi_scores_boot = pd.concat(fi_scores_boot_ls, axis=1)
            stability_results = {"scorer": [scoring_fn_name]}
            for metric_name, metric_fun in metrics.items():
                stability_results[metric_name] = [np.mean(pdist(fi_scores_boot.T, metric=metric_fun))]
            stability_dict[scoring_fn_name] = pd.DataFrame(stability_results)
        stability_df = pd.concat(stability_dict, axis=0).reset_index(drop=True)
        if single_scoring_fn:
            stability_df = stability_df.drop(columns=["scorer"])
        return stability_df

    def _fit_importance_scores(self, X, y):
        all_scores = []
        all_full_preds = []
        for estimator, transformer, tree_random_state in \
                zip(self.estimators, self.transformers, self.tree_random_states):
            tree_mdi_plus = TreeMDIPlus(estimator=estimator,
                                        transformer=transformer,
                                        scoring_fns=self.scoring_fns,
                                        sample_split=self.sample_split,
                                        tree_random_state=tree_random_state,
                                        mode=self.mode,
                                        task=self.task,
                                        center=self.center,
                                        normalize=self.normalize)
            scores = tree_mdi_plus.get_scores(X, y)
            if scores is not None:
                all_scores.append(scores)
                all_full_preds.append(tree_mdi_plus._full_preds)
        if len(all_scores) == 0:
            raise ValueError("Transformer representation was empty for all trees.")
        full_preds = np.nanmean(all_full_preds, axis=0)
        self._full_preds = full_preds
        scoring_fns = self.scoring_fns if isinstance(self.scoring_fns, dict) \
            else {"importance": self.scoring_fns}
        for fn_name, scoring_fn in scoring_fns.items():
            self.feature_importances_by_tree_[fn_name] = pd.concat([scores[fn_name] for scores in all_scores], axis=1)
            self.feature_importances_by_tree_[fn_name].columns = np.arange(len(all_scores))
            self.feature_importances_[fn_name] = np.mean(self.feature_importances_by_tree_[fn_name], axis=1)
            self.prediction_score_[fn_name] = [scoring_fn(y[~np.isnan(full_preds)], full_preds[~np.isnan(full_preds)])]
        if list(scoring_fns.keys()) == ["importance"]:
            self.prediction_score_ = self.prediction_score_["importance"]
            self.feature_importances_by_tree_ = self.feature_importances_by_tree_["importance"]
        if isinstance(X, pd.DataFrame):
            self.feature_importances_.index = X.columns
        self.feature_importances_.index.name = 'var'
        self.feature_importances_.reset_index(inplace=True)
        self.is_fitted = True

Methods

def get_scores(self, X, y)

Obtain the MDI+ feature importances for a forest.

Parameters

X : ndarray of shape (n_samples, n_features)
The covariate matrix. If a pd.DataFrame object is supplied, then the column names are used in the output
y : ndarray of shape (n_samples, n_targets)
The observed responses.

Returns

scores : pd.DataFrame of shape (n_features, n_scoring_fns)
The MDI+ feature importances.
Expand source code 
def get_scores(self, X, y):
    """
    Obtain the MDI+ feature importances for a forest.

    Parameters
    ----------
    X: ndarray of shape (n_samples, n_features)
        The covariate matrix. If a pd.DataFrame object is supplied, then
        the column names are used in the output
    y: ndarray of shape (n_samples, n_targets)
        The observed responses.

    Returns
    -------
    scores: pd.DataFrame of shape (n_features, n_scoring_fns)
        The MDI+ feature importances.
    """
    self._fit_importance_scores(X, y)
    return self.feature_importances_
def get_stability_scores(self, B=10, metrics='auto')

Evaluate the stability of the MDI+ feature importance rankings across bootstrapped samples of trees. Can be used to select the GLM and scoring metric in a data-driven manner, where the GLM and metric that yields the most stable feature rankings across bootstrapped samples is selected.

Parameters

B : int
Number of bootstrap samples.
metrics : "auto" or a dict with functions as value and function name (str) as key
Metric(s) used to evaluate the stability between two sets of feature importances. If "auto", then the feature importance stability metrics are: (1) Rank-based overlap (RBO) with p=0.9 (from "A Similarity Measure for Indefinite Rankings" by Webber et al. (2010)). Intuitively, this metric gives more weight to features with the largest importances, with most of the weight going to the ~1/(1-p) features with the largest importances. (2) A weighted kendall tau metric (tauAP_b from "The Treatment of Ties in AP Correlation" by Urbano and Marrero (2017)), which also gives more weight to the features with the largest importances, but uses a different weighting scheme from RBO. Note that these default metrics assume that a higher MDI+ score indicates greater importance and thus give more weight to these features with high importance/ranks. If a lower MDI+ score indicates higher importance, then invert either these stability metrics or the MDI+ scores before evaluating the stability.

Returns

stability_results : pd.DataFrame of shape (n_features, n_metrics)
The stability scores of the MDI+ feature rankings across bootstrapped samples.
Expand source code 
def get_stability_scores(self, B=10, metrics="auto"):
    """
    Evaluate the stability of the MDI+ feature importance rankings
    across bootstrapped samples of trees. Can be used to select the GLM
    and scoring metric in a data-driven manner, where the GLM and metric that
    yields the most stable feature rankings across bootstrapped samples is selected.

    Parameters
    ----------
    B: int
        Number of bootstrap samples.
    metrics: "auto" or a dict with functions as value and function name (str) as key
        Metric(s) used to evaluate the stability between two sets of feature importances.
        If "auto", then the feature importance stability metrics are:
            (1) Rank-based overlap (RBO) with p=0.9 (from "A Similarity Measure for
            Indefinite Rankings" by Webber et al. (2010)). Intuitively, this metric gives
            more weight to features with the largest importances, with most of the weight
            going to the ~1/(1-p) features with the largest importances.
            (2) A weighted kendall tau metric (tauAP_b from "The Treatment of Ties in
            AP Correlation" by Urbano and Marrero (2017)), which also gives more weight
            to the features with the largest importances, but uses a different weighting
            scheme from RBO.
        Note that these default metrics assume that a higher MDI+ score indicates
        greater importance and thus give more weight to these features with high
        importance/ranks. If a lower MDI+ score indicates higher importance, then invert
        either these stability metrics or the MDI+ scores before evaluating the stability.

    Returns
    -------
    stability_results: pd.DataFrame of shape (n_features, n_metrics)
        The stability scores of the MDI+ feature rankings across bootstrapped samples.

    """
    if metrics == "auto":
        metrics = {"RBO": partial(rbo, p=0.9), "tauAP": tauAP_b}
    elif not isinstance(metrics, dict):
        raise ValueError("`metrics` must be 'auto' or a dictionary "
                         "where the key is the metric name and the value is the evaluation function")
    single_scoring_fn = not isinstance(self.feature_importances_by_tree_, dict)
    if single_scoring_fn:
        feature_importances_dict = {"mdi_plus_score": self.feature_importances_by_tree_}
    else:
        feature_importances_dict = self.feature_importances_by_tree_
    stability_dict = {}
    for scoring_fn_name, feature_importances_by_tree in feature_importances_dict.items():
        n_trees = feature_importances_by_tree.shape[1]
        fi_scores_boot_ls = []
        for b in range(B):
            bootstrap_sample = np.random.choice(n_trees, n_trees, replace=True)
            fi_scores_boot_ls.append(feature_importances_by_tree[bootstrap_sample].mean(axis=1))
        fi_scores_boot = pd.concat(fi_scores_boot_ls, axis=1)
        stability_results = {"scorer": [scoring_fn_name]}
        for metric_name, metric_fun in metrics.items():
            stability_results[metric_name] = [np.mean(pdist(fi_scores_boot.T, metric=metric_fun))]
        stability_dict[scoring_fn_name] = pd.DataFrame(stability_results)
    stability_df = pd.concat(stability_dict, axis=0).reset_index(drop=True)
    if single_scoring_fn:
        stability_df = stability_df.drop(columns=["scorer"])
    return stability_df
class GenericClassifierPPM (estimator)

Partial prediction model for arbitrary classification estimators. May be slow.

Expand source code 
class GenericClassifierPPM(_GenericPPM, PartialPredictionModelBase, ABC):
    """
    Partial prediction model for arbitrary classification estimators. May be slow.
    """

    def predict_proba(self, X):
        return self.estimator.predict_proba(X)

    def predict_partial_k(self, blocked_data, k, mode):
        modified_data = blocked_data.get_modified_data(k, mode)
        return self.predict_proba(modified_data)

Ancestors

Methods

def predict_proba(self, X)
Expand source code 
def predict_proba(self, X):
    return self.estimator.predict_proba(X)

Inherited members

class GenericRegressorPPM (estimator)

Partial prediction model for arbitrary regression estimators. May be slow.

Expand source code 
class GenericRegressorPPM(_GenericPPM, PartialPredictionModelBase, ABC):
    """
    Partial prediction model for arbitrary regression estimators. May be slow.
    """
    ...

Ancestors

Inherited members

class GlmClassifierPPM (estimator, loo=True, alpha_grid=array([1.00000000e-04, 7.74263683e-04, 5.99484250e-03, 4.64158883e-02, 3.59381366e-01, 2.78255940e+00, 2.15443469e+01, 1.66810054e+02, 1.29154967e+03, 1.00000000e+04]), inv_link_fn=<function _GlmPPM.<lambda>>, l_dot=<function _GlmPPM.<lambda>>, l_doubledot=<function _GlmPPM.<lambda>>, r_doubledot=<function _GlmPPM.<lambda>>, hyperparameter_scorer=<function mean_squared_error>, trim=None, gcv_mode='auto')

PPM class for GLM classification estimator.

Expand source code 
class GlmClassifierPPM(_GlmPPM, PartialPredictionModelBase, ABC):
    """
    PPM class for GLM classification estimator.
    """

    def predict_proba(self, X):
        probs = self.predict(X)
        if probs.ndim == 1:
            probs = np.stack([1 - probs, probs], axis=1)
        return probs

    def predict_proba_loo(self, X):
        probs = self.predict_loo(X)
        if probs.ndim == 1:
            probs = np.stack([1 - probs, probs], axis=1)
        return probs

Ancestors

Subclasses

Methods

def predict_proba(self, X)
Expand source code 
def predict_proba(self, X):
    probs = self.predict(X)
    if probs.ndim == 1:
        probs = np.stack([1 - probs, probs], axis=1)
    return probs
def predict_proba_loo(self, X)
Expand source code 
def predict_proba_loo(self, X):
    probs = self.predict_loo(X)
    if probs.ndim == 1:
        probs = np.stack([1 - probs, probs], axis=1)
    return probs

Inherited members

class GlmRegressorPPM (estimator, loo=True, alpha_grid=array([1.00000000e-04, 7.74263683e-04, 5.99484250e-03, 4.64158883e-02, 3.59381366e-01, 2.78255940e+00, 2.15443469e+01, 1.66810054e+02, 1.29154967e+03, 1.00000000e+04]), inv_link_fn=<function _GlmPPM.<lambda>>, l_dot=<function _GlmPPM.<lambda>>, l_doubledot=<function _GlmPPM.<lambda>>, r_doubledot=<function _GlmPPM.<lambda>>, hyperparameter_scorer=<function mean_squared_error>, trim=None, gcv_mode='auto')

PPM class for GLM regression estimator.

Expand source code 
class GlmRegressorPPM(_GlmPPM, PartialPredictionModelBase, ABC):
    """
    PPM class for GLM regression estimator.
    """
    ...

Ancestors

Subclasses

Inherited members

class IdentityTransformer

Block transformer that creates a block partitioned data object with each block k containing only the original feature k.

Expand source code 
class IdentityTransformer(BlockTransformerBase, ABC):
    """
    Block transformer that creates a block partitioned data object with each
    block k containing only the original feature k.
    """

    def _fit_one_feature(self, X, k):
        self._centers[k] = np.mean(X[:, [k]])
        self._scales[k] = np.std(X[:, [k]])

    def _transform_one_feature(self, X, k):
        return X[:, [k]]

Ancestors

Inherited members

class LassoRegressorPPM (loo=True, alpha_grid=array([1.00000000e-02, 1.61559810e-02, 2.61015722e-02, 4.21696503e-02, 6.81292069e-02, 1.10069417e-01, 1.77827941e-01, 2.87298483e-01, 4.64158883e-01, 7.49894209e-01, 1.21152766e+00, 1.95734178e+00, 3.16227766e+00, 5.10896977e+00, 8.25404185e+00, 1.33352143e+01, 2.15443469e+01, 3.48070059e+01, 5.62341325e+01, 9.08517576e+01, 1.46779927e+02, 2.37137371e+02, 3.83118685e+02, 6.18965819e+02, 1.00000000e+03]), **kwargs)

PPM class for regression that uses lasso as the estimator.

Parameters

loo : bool
Flag for whether to also use LOO calculations for making predictions.
alpha_grid : ndarray of shape (n_alphas, )
The grid of alpha values for hyperparameter optimization.
**kwargs
Other Parameters are passed on to Lasso().
Expand source code 
class LassoRegressorPPM(GlmRegressorPPM, PartialPredictionModelBase, ABC):
    """
    PPM class for regression that uses lasso as the estimator.

    Parameters
    ----------
    loo: bool
        Flag for whether to also use LOO calculations for making predictions.
    alpha_grid: ndarray of shape (n_alphas, )
        The grid of alpha values for hyperparameter optimization.
    **kwargs
        Other Parameters are passed on to Lasso().
    """

    def __init__(self, loo=True, alpha_grid=np.logspace(-2, 3, 25), **kwargs):
        super().__init__(Lasso(**kwargs), loo, alpha_grid, r_doubledot=None)

Ancestors

Inherited members

class LogisticClassifierPPM (loo=True, alpha_grid=array([1.00000000e-02, 1.61559810e-02, 2.61015722e-02, 4.21696503e-02, 6.81292069e-02, 1.10069417e-01, 1.77827941e-01, 2.87298483e-01, 4.64158883e-01, 7.49894209e-01, 1.21152766e+00, 1.95734178e+00, 3.16227766e+00, 5.10896977e+00, 8.25404185e+00, 1.33352143e+01, 2.15443469e+01, 3.48070059e+01, 5.62341325e+01, 9.08517576e+01, 1.46779927e+02, 2.37137371e+02, 3.83118685e+02, 6.18965819e+02, 1.00000000e+03]), penalty='l2', max_iter=1000, trim=0.01, **kwargs)

PPM class for classification that uses logistic regression as the estimator.

Parameters

loo : bool
Flag for whether to also use LOO calculations for making predictions.
alpha_grid : ndarray of shape (n_alphas, )
The grid of alpha values for hyperparameter optimization.
max_iter : int
The maximum number of iterations for the LogisticRegression solver.
trim : float
The amount by which to trim predicted probabilities away from 0 and 1. This helps to stabilize some loss calculations.
**kwargs
Other Parameters are passed on to LogisticRegression().
Expand source code 
class LogisticClassifierPPM(GlmClassifierPPM, PartialPredictionModelBase, ABC):
    """
    PPM class for classification that uses logistic regression as the estimator.

    Parameters
    ----------
    loo: bool
        Flag for whether to also use LOO calculations for making predictions.
    alpha_grid: ndarray of shape (n_alphas, )
        The grid of alpha values for hyperparameter optimization.
    max_iter: int
        The maximum number of iterations for the LogisticRegression solver.
    trim: float
        The amount by which to trim predicted probabilities away from 0 and 1.
        This helps to stabilize some loss calculations.
    **kwargs
        Other Parameters are passed on to LogisticRegression().
    """

    def __init__(self, loo=True, alpha_grid=np.logspace(-2, 3, 25),
                 penalty='l2', max_iter=1000, trim=0.01, **kwargs):
        assert penalty in ['l2', 'l1']
        if penalty == 'l2':
            r_doubledot = lambda a: 1
        elif penalty == 'l1':
            r_doubledot = None
        super().__init__(LogisticRegression(penalty=penalty, max_iter=max_iter, **kwargs),
                         loo, alpha_grid,
                         inv_link_fn=sp.special.expit,
                         l_doubledot=lambda a, b: b * (1 - b),
                         r_doubledot=r_doubledot,
                         hyperparameter_scorer=log_loss,
                         trim=trim)

Ancestors

Inherited members

class MDIPlusDefaultTransformer (tree_model, rescale_mode='max', drop_features=True)

Default block transformer used in MDI+. For each original feature, this forms a block comprising the local decision stumps, from a single tree model, that split on the feature, and appends the original feature.

Parameters

tree_model : scikit-learn estimator
The scikit-learn tree estimator object.
rescale_mode : string in {"max", "mean", None}
Flag for the type of rescaling to be done to the blocks from different base transformers. If "max", divide each block by the max std deviation of a column within the block. If "mean", divide each block by the mean std deviation of a column within the block. If None, do not rescale.
drop_features : bool
Flag for whether to return an empty block if that from the first transformer in the list is trivial.
Expand source code 
class MDIPlusDefaultTransformer(CompositeTransformer, ABC):
    """
    Default block transformer used in MDI+. For each original feature, this
    forms a block comprising the local decision stumps, from a single tree
    model, that split on the feature, and appends the original feature.

    Parameters
    ----------
    tree_model: scikit-learn estimator
        The scikit-learn tree estimator object.
    rescale_mode: string in {"max", "mean", None}
        Flag for the type of rescaling to be done to the blocks from different
        base transformers. If "max", divide each block by the max std deviation
        of a column within the block. If "mean", divide each block by the mean
        std deviation of a column within the block. If None, do not rescale.
    drop_features: bool
        Flag for whether to return an empty block if that from the first
        transformer in the list is trivial.
    """
    def __init__(self, tree_model, rescale_mode="max", drop_features=True):
        super().__init__([TreeTransformer(tree_model), IdentityTransformer()],
                         rescale_mode, drop_features)

Ancestors

Inherited members

class RandomForestPlusClassifier (rf_model=None, prediction_model=None, sample_split='auto', include_raw=True, drop_features=True, add_transformers=None, center=True, normalize=False)

The class object for the Random Forest Plus (RF+) classification estimator, which can be used as a prediction model or interpreted via generalized mean decrease in impurity (MDI+). For more details, refer to [paper].

Expand source code 
class RandomForestPlusClassifier(_RandomForestPlus, ClassifierMixin):
    """
    The class object for the Random Forest Plus (RF+) classification estimator, which can
    be used as a prediction model or interpreted via generalized
    mean decrease in impurity (MDI+). For more details, refer to [paper].
    """
    ...

Ancestors

  • imodels.importance.rf_plus._RandomForestPlus
  • sklearn.base.BaseEstimator
  • sklearn.utils._estimator_html_repr._HTMLDocumentationLinkMixin
  • sklearn.utils._metadata_requests._MetadataRequester
  • sklearn.base.ClassifierMixin

Methods

def set_fit_request(self: RandomForestPlusClassifier, *, sample_weight: bool | str | None = '$UNCHANGED$') ‑> RandomForestPlusClassifier

Request metadata passed to the fit method.

Note that this method is only relevant if enable_metadata_routing=True (see :func:sklearn.set_config). Please see :ref:User Guide <metadata_routing> on how the routing mechanism works.

The options for each parameter are:

  • True: metadata is requested, and passed to fit if provided. The request is ignored if metadata is not provided.

  • False: metadata is not requested and the meta-estimator will not pass it to fit.

  • None: metadata is not requested, and the meta-estimator will raise an error if the user provides it.

  • str: metadata should be passed to the meta-estimator with this given alias instead of the original name.

The default (sklearn.utils.metadata_routing.UNCHANGED) retains the existing request. This allows you to change the request for some parameters and not others.

Added in version: 1.3

Note

This method is only relevant if this estimator is used as a sub-estimator of a meta-estimator, e.g. used inside a :class:~sklearn.pipeline.Pipeline. Otherwise it has no effect.

Parameters

sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
Metadata routing for sample_weight parameter in fit.

Returns

self : object
The updated object.
Expand source code 
def func(*args, **kw):
    """Updates the request for provided parameters

    This docstring is overwritten below.
    See REQUESTER_DOC for expected functionality
    """
    if not _routing_enabled():
        raise RuntimeError(
            "This method is only available when metadata routing is enabled."
            " You can enable it using"
            " sklearn.set_config(enable_metadata_routing=True)."
        )

    if self.validate_keys and (set(kw) - set(self.keys)):
        raise TypeError(
            f"Unexpected args: {set(kw) - set(self.keys)} in {self.name}. "
            f"Accepted arguments are: {set(self.keys)}"
        )

    # This makes it possible to use the decorated method as an unbound method,
    # for instance when monkeypatching.
    # https://github.com/scikit-learn/scikit-learn/issues/28632
    if instance is None:
        _instance = args[0]
        args = args[1:]
    else:
        _instance = instance

    # Replicating python's behavior when positional args are given other than
    # `self`, and `self` is only allowed if this method is unbound.
    if args:
        raise TypeError(
            f"set_{self.name}_request() takes 0 positional argument but"
            f" {len(args)} were given"
        )

    requests = _instance._get_metadata_request()
    method_metadata_request = getattr(requests, self.name)

    for prop, alias in kw.items():
        if alias is not UNCHANGED:
            method_metadata_request.add_request(param=prop, alias=alias)
    _instance._metadata_request = requests

    return _instance
def set_score_request(self: RandomForestPlusClassifier, *, sample_weight: bool | str | None = '$UNCHANGED$') ‑> RandomForestPlusClassifier

Request metadata passed to the score method.

Note that this method is only relevant if enable_metadata_routing=True (see :func:sklearn.set_config). Please see :ref:User Guide <metadata_routing> on how the routing mechanism works.

The options for each parameter are:

  • True: metadata is requested, and passed to score if provided. The request is ignored if metadata is not provided.

  • False: metadata is not requested and the meta-estimator will not pass it to score.

  • None: metadata is not requested, and the meta-estimator will raise an error if the user provides it.

  • str: metadata should be passed to the meta-estimator with this given alias instead of the original name.

The default (sklearn.utils.metadata_routing.UNCHANGED) retains the existing request. This allows you to change the request for some parameters and not others.

Added in version: 1.3

Note

This method is only relevant if this estimator is used as a sub-estimator of a meta-estimator, e.g. used inside a :class:~sklearn.pipeline.Pipeline. Otherwise it has no effect.

Parameters

sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
Metadata routing for sample_weight parameter in score.

Returns

self : object
The updated object.
Expand source code 
def func(*args, **kw):
    """Updates the request for provided parameters

    This docstring is overwritten below.
    See REQUESTER_DOC for expected functionality
    """
    if not _routing_enabled():
        raise RuntimeError(
            "This method is only available when metadata routing is enabled."
            " You can enable it using"
            " sklearn.set_config(enable_metadata_routing=True)."
        )

    if self.validate_keys and (set(kw) - set(self.keys)):
        raise TypeError(
            f"Unexpected args: {set(kw) - set(self.keys)} in {self.name}. "
            f"Accepted arguments are: {set(self.keys)}"
        )

    # This makes it possible to use the decorated method as an unbound method,
    # for instance when monkeypatching.
    # https://github.com/scikit-learn/scikit-learn/issues/28632
    if instance is None:
        _instance = args[0]
        args = args[1:]
    else:
        _instance = instance

    # Replicating python's behavior when positional args are given other than
    # `self`, and `self` is only allowed if this method is unbound.
    if args:
        raise TypeError(
            f"set_{self.name}_request() takes 0 positional argument but"
            f" {len(args)} were given"
        )

    requests = _instance._get_metadata_request()
    method_metadata_request = getattr(requests, self.name)

    for prop, alias in kw.items():
        if alias is not UNCHANGED:
            method_metadata_request.add_request(param=prop, alias=alias)
    _instance._metadata_request = requests

    return _instance
class RandomForestPlusRegressor (rf_model=None, prediction_model=None, sample_split='auto', include_raw=True, drop_features=True, add_transformers=None, center=True, normalize=False)

The class object for the Random Forest Plus (RF+) regression estimator, which can be used as a prediction model or interpreted via generalized mean decrease in impurity (MDI+). For more details, refer to [paper].

Expand source code 
class RandomForestPlusRegressor(_RandomForestPlus, RegressorMixin):
    """
    The class object for the Random Forest Plus (RF+) regression estimator, which can
    be used as a prediction model or interpreted via generalized
    mean decrease in impurity (MDI+). For more details, refer to [paper].
    """
    ...

Ancestors

  • imodels.importance.rf_plus._RandomForestPlus
  • sklearn.base.BaseEstimator
  • sklearn.utils._estimator_html_repr._HTMLDocumentationLinkMixin
  • sklearn.utils._metadata_requests._MetadataRequester
  • sklearn.base.RegressorMixin

Methods

def set_fit_request(self: RandomForestPlusRegressor, *, sample_weight: bool | str | None = '$UNCHANGED$') ‑> RandomForestPlusRegressor

Request metadata passed to the fit method.

Note that this method is only relevant if enable_metadata_routing=True (see :func:sklearn.set_config). Please see :ref:User Guide <metadata_routing> on how the routing mechanism works.

The options for each parameter are:

  • True: metadata is requested, and passed to fit if provided. The request is ignored if metadata is not provided.

  • False: metadata is not requested and the meta-estimator will not pass it to fit.

  • None: metadata is not requested, and the meta-estimator will raise an error if the user provides it.

  • str: metadata should be passed to the meta-estimator with this given alias instead of the original name.

The default (sklearn.utils.metadata_routing.UNCHANGED) retains the existing request. This allows you to change the request for some parameters and not others.

Added in version: 1.3

Note

This method is only relevant if this estimator is used as a sub-estimator of a meta-estimator, e.g. used inside a :class:~sklearn.pipeline.Pipeline. Otherwise it has no effect.

Parameters

sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
Metadata routing for sample_weight parameter in fit.

Returns

self : object
The updated object.
Expand source code 
def func(*args, **kw):
    """Updates the request for provided parameters

    This docstring is overwritten below.
    See REQUESTER_DOC for expected functionality
    """
    if not _routing_enabled():
        raise RuntimeError(
            "This method is only available when metadata routing is enabled."
            " You can enable it using"
            " sklearn.set_config(enable_metadata_routing=True)."
        )

    if self.validate_keys and (set(kw) - set(self.keys)):
        raise TypeError(
            f"Unexpected args: {set(kw) - set(self.keys)} in {self.name}. "
            f"Accepted arguments are: {set(self.keys)}"
        )

    # This makes it possible to use the decorated method as an unbound method,
    # for instance when monkeypatching.
    # https://github.com/scikit-learn/scikit-learn/issues/28632
    if instance is None:
        _instance = args[0]
        args = args[1:]
    else:
        _instance = instance

    # Replicating python's behavior when positional args are given other than
    # `self`, and `self` is only allowed if this method is unbound.
    if args:
        raise TypeError(
            f"set_{self.name}_request() takes 0 positional argument but"
            f" {len(args)} were given"
        )

    requests = _instance._get_metadata_request()
    method_metadata_request = getattr(requests, self.name)

    for prop, alias in kw.items():
        if alias is not UNCHANGED:
            method_metadata_request.add_request(param=prop, alias=alias)
    _instance._metadata_request = requests

    return _instance
def set_score_request(self: RandomForestPlusRegressor, *, sample_weight: bool | str | None = '$UNCHANGED$') ‑> RandomForestPlusRegressor

Request metadata passed to the score method.

Note that this method is only relevant if enable_metadata_routing=True (see :func:sklearn.set_config). Please see :ref:User Guide <metadata_routing> on how the routing mechanism works.

The options for each parameter are:

  • True: metadata is requested, and passed to score if provided. The request is ignored if metadata is not provided.

  • False: metadata is not requested and the meta-estimator will not pass it to score.

  • None: metadata is not requested, and the meta-estimator will raise an error if the user provides it.

  • str: metadata should be passed to the meta-estimator with this given alias instead of the original name.

The default (sklearn.utils.metadata_routing.UNCHANGED) retains the existing request. This allows you to change the request for some parameters and not others.

Added in version: 1.3

Note

This method is only relevant if this estimator is used as a sub-estimator of a meta-estimator, e.g. used inside a :class:~sklearn.pipeline.Pipeline. Otherwise it has no effect.

Parameters

sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
Metadata routing for sample_weight parameter in score.

Returns

self : object
The updated object.
Expand source code 
def func(*args, **kw):
    """Updates the request for provided parameters

    This docstring is overwritten below.
    See REQUESTER_DOC for expected functionality
    """
    if not _routing_enabled():
        raise RuntimeError(
            "This method is only available when metadata routing is enabled."
            " You can enable it using"
            " sklearn.set_config(enable_metadata_routing=True)."
        )

    if self.validate_keys and (set(kw) - set(self.keys)):
        raise TypeError(
            f"Unexpected args: {set(kw) - set(self.keys)} in {self.name}. "
            f"Accepted arguments are: {set(self.keys)}"
        )

    # This makes it possible to use the decorated method as an unbound method,
    # for instance when monkeypatching.
    # https://github.com/scikit-learn/scikit-learn/issues/28632
    if instance is None:
        _instance = args[0]
        args = args[1:]
    else:
        _instance = instance

    # Replicating python's behavior when positional args are given other than
    # `self`, and `self` is only allowed if this method is unbound.
    if args:
        raise TypeError(
            f"set_{self.name}_request() takes 0 positional argument but"
            f" {len(args)} were given"
        )

    requests = _instance._get_metadata_request()
    method_metadata_request = getattr(requests, self.name)

    for prop, alias in kw.items():
        if alias is not UNCHANGED:
            method_metadata_request.add_request(param=prop, alias=alias)
    _instance._metadata_request = requests

    return _instance
class RidgeClassifierPPM (loo=True, alpha_grid=array([1.00000000e-05, 1.26185688e-05, 1.59228279e-05, 2.00923300e-05, 2.53536449e-05, 3.19926714e-05, 4.03701726e-05, 5.09413801e-05, 6.42807312e-05, 8.11130831e-05, 1.02353102e-04, 1.29154967e-04, 1.62975083e-04, 2.05651231e-04, 2.59502421e-04, 3.27454916e-04, 4.13201240e-04, 5.21400829e-04, 6.57933225e-04, 8.30217568e-04, 1.04761575e-03, 1.32194115e-03, 1.66810054e-03, 2.10490414e-03, 2.65608778e-03, 3.35160265e-03, 4.22924287e-03, 5.33669923e-03, 6.73415066e-03, 8.49753436e-03, 1.07226722e-02, 1.35304777e-02, 1.70735265e-02, 2.15443469e-02, 2.71858824e-02, 3.43046929e-02, 4.32876128e-02, 5.46227722e-02, 6.89261210e-02, 8.69749003e-02, 1.09749877e-01, 1.38488637e-01, 1.74752840e-01, 2.20513074e-01, 2.78255940e-01, 3.51119173e-01, 4.43062146e-01, 5.59081018e-01, 7.05480231e-01, 8.90215085e-01, 1.12332403e+00, 1.41747416e+00, 1.78864953e+00, 2.25701972e+00, 2.84803587e+00, 3.59381366e+00, 4.53487851e+00, 5.72236766e+00, 7.22080902e+00, 9.11162756e+00, 1.14975700e+01, 1.45082878e+01, 1.83073828e+01, 2.31012970e+01, 2.91505306e+01, 3.67837977e+01, 4.64158883e+01, 5.85702082e+01, 7.39072203e+01, 9.32603347e+01, 1.17681195e+02, 1.48496826e+02, 1.87381742e+02, 2.36448941e+02, 2.98364724e+02, 3.76493581e+02, 4.75081016e+02, 5.99484250e+02, 7.56463328e+02, 9.54548457e+02, 1.20450354e+03, 1.51991108e+03, 1.91791026e+03, 2.42012826e+03, 3.05385551e+03, 3.85352859e+03, 4.86260158e+03, 6.13590727e+03, 7.74263683e+03, 9.77009957e+03, 1.23284674e+04, 1.55567614e+04, 1.96304065e+04, 2.47707636e+04, 3.12571585e+04, 3.94420606e+04, 4.97702356e+04, 6.28029144e+04, 7.92482898e+04, 1.00000000e+05]), gcv_mode='auto', **kwargs)

PPM class for classification that uses ridge as the GLM estimator.

Expand source code 
class RidgeClassifierPPM(_RidgePPM, GlmClassifierPPM,
                         PartialPredictionModelBase, ABC):
    """
    PPM class for classification that uses ridge as the GLM estimator.
    """

    def predict_proba(self, X):
        probs = softmax(self.predict(X))
        if probs.ndim == 1:
            probs = np.stack([1 - probs, probs], axis=1)
        return probs

    def predict_proba_loo(self, X):
        probs = softmax(self.predict_loo(X))
        if probs.ndim == 1:
            probs = np.stack([1 - probs, probs], axis=1)
        return probs

Ancestors

Methods

def predict_proba(self, X)
Expand source code 
def predict_proba(self, X):
    probs = softmax(self.predict(X))
    if probs.ndim == 1:
        probs = np.stack([1 - probs, probs], axis=1)
    return probs
def predict_proba_loo(self, X)
Expand source code 
def predict_proba_loo(self, X):
    probs = softmax(self.predict_loo(X))
    if probs.ndim == 1:
        probs = np.stack([1 - probs, probs], axis=1)
    return probs

Inherited members

class RidgeRegressorPPM (loo=True, alpha_grid=array([1.00000000e-05, 1.26185688e-05, 1.59228279e-05, 2.00923300e-05, 2.53536449e-05, 3.19926714e-05, 4.03701726e-05, 5.09413801e-05, 6.42807312e-05, 8.11130831e-05, 1.02353102e-04, 1.29154967e-04, 1.62975083e-04, 2.05651231e-04, 2.59502421e-04, 3.27454916e-04, 4.13201240e-04, 5.21400829e-04, 6.57933225e-04, 8.30217568e-04, 1.04761575e-03, 1.32194115e-03, 1.66810054e-03, 2.10490414e-03, 2.65608778e-03, 3.35160265e-03, 4.22924287e-03, 5.33669923e-03, 6.73415066e-03, 8.49753436e-03, 1.07226722e-02, 1.35304777e-02, 1.70735265e-02, 2.15443469e-02, 2.71858824e-02, 3.43046929e-02, 4.32876128e-02, 5.46227722e-02, 6.89261210e-02, 8.69749003e-02, 1.09749877e-01, 1.38488637e-01, 1.74752840e-01, 2.20513074e-01, 2.78255940e-01, 3.51119173e-01, 4.43062146e-01, 5.59081018e-01, 7.05480231e-01, 8.90215085e-01, 1.12332403e+00, 1.41747416e+00, 1.78864953e+00, 2.25701972e+00, 2.84803587e+00, 3.59381366e+00, 4.53487851e+00, 5.72236766e+00, 7.22080902e+00, 9.11162756e+00, 1.14975700e+01, 1.45082878e+01, 1.83073828e+01, 2.31012970e+01, 2.91505306e+01, 3.67837977e+01, 4.64158883e+01, 5.85702082e+01, 7.39072203e+01, 9.32603347e+01, 1.17681195e+02, 1.48496826e+02, 1.87381742e+02, 2.36448941e+02, 2.98364724e+02, 3.76493581e+02, 4.75081016e+02, 5.99484250e+02, 7.56463328e+02, 9.54548457e+02, 1.20450354e+03, 1.51991108e+03, 1.91791026e+03, 2.42012826e+03, 3.05385551e+03, 3.85352859e+03, 4.86260158e+03, 6.13590727e+03, 7.74263683e+03, 9.77009957e+03, 1.23284674e+04, 1.55567614e+04, 1.96304065e+04, 2.47707636e+04, 3.12571585e+04, 3.94420606e+04, 4.97702356e+04, 6.28029144e+04, 7.92482898e+04, 1.00000000e+05]), gcv_mode='auto', **kwargs)

PPM class for regression that uses ridge as the GLM estimator.

Expand source code 
class RidgeRegressorPPM(_RidgePPM, GlmRegressorPPM,
                        PartialPredictionModelBase, ABC):
    """
    PPM class for regression that uses ridge as the GLM estimator.
    """
    ...

Ancestors

Inherited members

class RobustRegressorPPM (loo=True, alpha_grid=array([1.00000000e-02, 1.61559810e-02, 2.61015722e-02, 4.21696503e-02, 6.81292069e-02, 1.10069417e-01, 1.77827941e-01, 2.87298483e-01, 4.64158883e-01, 7.49894209e-01, 1.21152766e+00, 1.95734178e+00, 3.16227766e+00, 5.10896977e+00, 8.25404185e+00, 1.33352143e+01, 2.15443469e+01, 3.48070059e+01, 5.62341325e+01, 9.08517576e+01, 1.46779927e+02, 2.37137371e+02, 3.83118685e+02, 6.18965819e+02, 1.00000000e+03]), epsilon=1.35, max_iter=2000, **kwargs)

PPM class for regression that uses Huber robust regression as the estimator.

Parameters

loo : bool
Flag for whether to also use LOO calculations for making predictions.
alpha_grid : ndarray of shape (n_alphas, )
The grid of alpha values for hyperparameter optimization.
epsilon : float
The robustness parameter for Huber regression. The smaller the epsilon, the more robust it is to outliers. Epsilon must be in the range [1, inf).
**kwargs
Other Parameters are passed on to LogisticRegression().
Expand source code 
class RobustRegressorPPM(GlmRegressorPPM, PartialPredictionModelBase, ABC):
    """
    PPM class for regression that uses Huber robust regression as the estimator.

    Parameters
    ----------
    loo: bool
        Flag for whether to also use LOO calculations for making predictions.
    alpha_grid: ndarray of shape (n_alphas, )
        The grid of alpha values for hyperparameter optimization.
    epsilon: float
        The robustness parameter for Huber regression. The smaller the epsilon,
        the more robust it is to outliers. Epsilon must be in the range
        [1, inf).
    **kwargs
        Other Parameters are passed on to LogisticRegression().
    """
    def __init__(self, loo=True, alpha_grid=np.logspace(-2, 3, 25),
                 epsilon=1.35, max_iter=2000, **kwargs):
        loss_fn = partial(huber_loss, epsilon=epsilon)
        l_dot = lambda a, b: (b - a) / (1 + ((a - b) / epsilon) ** 2) ** 0.5
        l_doubledot=lambda a, b: (1 + (((a - b) / epsilon) ** 2)) ** (-1.5)
        super().__init__(
            HuberRegressor(max_iter=max_iter, **kwargs), loo, alpha_grid,
            l_dot=l_dot,
            l_doubledot=l_doubledot,
            hyperparameter_scorer=loss_fn)

Ancestors

Inherited members

class TreeMDIPlus (estimator, transformer, scoring_fns, sample_split='loo', tree_random_state=None, mode='keep_k', task='regression', center=True, normalize=False)

The class object for computing MDI+ feature importances for a single tree. Generalized mean decrease in impurity (MDI+) is a flexible framework for computing RF feature importances. For more details, refer to [paper].

Parameters

estimator : a fitted PartialPredictionModelBase object or scikit-learn type estimator
The fitted partial prediction model to use for evaluating feature importance via MDI+. If not a PartialPredictionModelBase, then the estimator is coerced into a PartialPredictionModelBase object via GenericRegressorPPM or GenericClassifierPPM depending on the specified task. Note that these generic PPMs may be computationally expensive.
transformer : a BlockTransformerBase object
A block feature transformer used to generate blocks of engineered features for each original feature. The transformed data is then used as input into the partial prediction models.
scoring_fns : a function or dict with functions as value and function name (str) as key
The scoring functions used for evaluating the partial predictions.
sample_split : string in {"loo", "oob", "inbag"} or None
The sample splitting strategy to be used when evaluating the partial model predictions. The default "loo" (leave-one-out) is strongly recommended for performance and in particular, for overcoming the known correlation and entropy biases suffered by MDI. "oob" (out-of-bag) can also be used to overcome these biases. "inbag" is the sample splitting strategy used by MDI. If None, no sample splitting is performed and the full data set is used to evaluate the partial model predictions.
tree_random_state : int or None
Random state of the fitted tree; used in sample splitting and only required if sample_split = "oob" or "inbag".
mode : string in {"keep_k", "keep_rest"}
Mode for the method. "keep_k" imputes the mean of each feature not in block k when making a partial model prediction, while "keep_rest" imputes the mean of each feature in block k. "keep_k" is strongly recommended for computational considerations.
task : string in {"regression", "classification"}
The supervised learning task for the RF model. Used for choosing defaults for the scoring_fns. Currently only regression and classification are supported.
center : bool
Flag for whether to center the transformed data in the transformers.
normalize : bool
Flag for whether to rescale the transformed data to have unit variance in the transformers.
Expand source code 
class TreeMDIPlus:
    """
    The class object for computing MDI+ feature importances for a single tree.
    Generalized mean decrease in impurity (MDI+) is a flexible framework for computing RF
    feature importances. For more details, refer to [paper].

    Parameters
    ----------
    estimator: a fitted PartialPredictionModelBase object or scikit-learn type estimator
        The fitted partial prediction model to use for evaluating
        feature importance via MDI+. If not a PartialPredictionModelBase, then
        the estimator is coerced into a PartialPredictionModelBase object via
        GenericRegressorPPM or GenericClassifierPPM depending on the specified
        task. Note that these generic PPMs may be computationally expensive.
    transformer: a BlockTransformerBase object
        A block feature transformer used to generate blocks of engineered
        features for each original feature. The transformed data is then used
        as input into the partial prediction models.
    scoring_fns: a function or dict with functions as value and function name (str) as key
        The scoring functions used for evaluating the partial predictions.
    sample_split: string in {"loo", "oob", "inbag"} or None
        The sample splitting strategy to be used when evaluating the partial
        model predictions. The default "loo" (leave-one-out) is strongly
        recommended for performance and in particular, for overcoming the known
        correlation and entropy biases suffered by MDI. "oob" (out-of-bag) can
        also be used to overcome these biases. "inbag" is the sample splitting
        strategy used by MDI. If None, no sample splitting is performed and the
        full data set is used to evaluate the partial model predictions.
    tree_random_state: int or None
        Random state of the fitted tree; used in sample splitting and
        only required if sample_split = "oob" or "inbag".
    mode: string in {"keep_k", "keep_rest"}
        Mode for the method. "keep_k" imputes the mean of each feature not
        in block k when making a partial model prediction, while "keep_rest"
        imputes the mean of each feature in block k. "keep_k" is strongly
        recommended for computational considerations.
    task: string in {"regression", "classification"}
        The supervised learning task for the RF model. Used for choosing
        defaults for the scoring_fns. Currently only regression and
        classification are supported.
    center: bool
        Flag for whether to center the transformed data in the transformers.
    normalize: bool
        Flag for whether to rescale the transformed data to have unit
        variance in the transformers.
    """

    def __init__(self, estimator, transformer, scoring_fns,
                 sample_split="loo", tree_random_state=None, mode="keep_k",
                 task="regression", center=True, normalize=False):
        assert sample_split in ["loo", "oob", "inbag", "auto", None]
        assert mode in ["keep_k", "keep_rest"]
        assert task in ["regression", "classification"]
        self.estimator = estimator
        self.transformer = transformer
        self.scoring_fns = scoring_fns
        self.sample_split = sample_split
        self.tree_random_state = tree_random_state
        _validate_sample_split(self.sample_split, self.estimator, isinstance(self.estimator, PartialPredictionModelBase))
        if self.sample_split in ["oob", "inbag"] and not self.tree_random_state:
            raise ValueError("Must specify tree_random_state to use 'oob' or 'inbag' sample_split.")
        self.mode = mode
        self.task = task
        self.center = center
        self.normalize = normalize
        self.is_fitted = False
        self._full_preds = None
        self.prediction_score_ = None
        self.feature_importances_ = None

    def get_scores(self, X, y):
        """
        Obtain the MDI+ feature importances for a single tree.

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            The covariate matrix. If a pd.DataFrame object is supplied, then
            the column names are used in the output
        y: ndarray of shape (n_samples, n_targets)
            The observed responses.

        Returns
        -------
        scores: pd.DataFrame of shape (n_features, n_scoring_fns)
            The MDI+ feature importances.
        """
        self._fit_importance_scores(X, y)
        return self.feature_importances_

    def _fit_importance_scores(self, X, y):
        n_samples = y.shape[0]
        blocked_data = self.transformer.transform(X, center=self.center,
                                                  normalize=self.normalize)
        self.n_features = blocked_data.n_blocks
        train_blocked_data, test_blocked_data, y_train, y_test, test_indices = \
            _get_sample_split_data(blocked_data, y, self.tree_random_state, self.sample_split)
        if train_blocked_data.get_all_data().shape[1] != 0:
            if hasattr(self.estimator, "predict_full") and \
                    hasattr(self.estimator, "predict_partial"):
                full_preds = self.estimator.predict_full(test_blocked_data)
                partial_preds = self.estimator.predict_partial(test_blocked_data, mode=self.mode)
            else:
                if self.task == "regression":
                    ppm = GenericRegressorPPM(self.estimator)
                elif self.task == "classification":
                    ppm = GenericClassifierPPM(self.estimator)
                full_preds = ppm.predict_full(test_blocked_data)
                partial_preds = ppm.predict_partial(test_blocked_data, mode=self.mode)
            self._score_full_predictions(y_test, full_preds)
            self._score_partial_predictions(y_test, full_preds, partial_preds)

            full_preds_n = np.empty(n_samples) if full_preds.ndim == 1 \
                else np.empty((n_samples, full_preds.shape[1]))
            full_preds_n[:] = np.nan
            full_preds_n[test_indices] = full_preds
            self._full_preds = full_preds_n
        self.is_fitted = True

    def _score_full_predictions(self, y_test, full_preds):
        scoring_fns = self.scoring_fns if isinstance(self.scoring_fns, dict) \
            else {"score": self.scoring_fns}
        all_prediction_scores = pd.DataFrame({})
        for fn_name, scoring_fn in scoring_fns.items():
            scores = scoring_fn(y_test, full_preds)
            all_prediction_scores[fn_name] = [scores]
        self.prediction_score_ = all_prediction_scores

    def _score_partial_predictions(self, y_test, full_preds, partial_preds):
        scoring_fns = self.scoring_fns if isinstance(self.scoring_fns, dict) \
            else {"importance": self.scoring_fns}
        all_scores = pd.DataFrame({})
        for fn_name, scoring_fn in scoring_fns.items():
            scores = _partial_preds_to_scores(partial_preds, y_test, scoring_fn)
            if self.mode == "keep_rest":
                full_score = scoring_fn(y_test, full_preds)
                scores = full_score - scores
            if len(partial_preds) != scores.size:
                if len(scoring_fns) > 1:
                    msg = "scoring_fn={} should return one value for each feature.".format(fn_name)
                else:
                    msg = "scoring_fns should return one value for each feature."
                raise ValueError("Unexpected dimensions. {}".format(msg))
            scores = scores.ravel()
            all_scores[fn_name] = scores
        self.feature_importances_ = all_scores

Methods

def get_scores(self, X, y)

Obtain the MDI+ feature importances for a single tree.

Parameters

X : ndarray of shape (n_samples, n_features)
The covariate matrix. If a pd.DataFrame object is supplied, then the column names are used in the output
y : ndarray of shape (n_samples, n_targets)
The observed responses.

Returns

scores : pd.DataFrame of shape (n_features, n_scoring_fns)
The MDI+ feature importances.
Expand source code 
def get_scores(self, X, y):
    """
    Obtain the MDI+ feature importances for a single tree.

    Parameters
    ----------
    X: ndarray of shape (n_samples, n_features)
        The covariate matrix. If a pd.DataFrame object is supplied, then
        the column names are used in the output
    y: ndarray of shape (n_samples, n_targets)
        The observed responses.

    Returns
    -------
    scores: pd.DataFrame of shape (n_features, n_scoring_fns)
        The MDI+ feature importances.
    """
    self._fit_importance_scores(X, y)
    return self.feature_importances_
class TreeTransformer (estimator, data=None)

A block transformer that transforms data using a representation built from local decision stumps from a tree or tree ensemble. The transformer also comes with metadata on the local decision stumps and methods that allow for transformations using sub-representations corresponding to each of the original features.

Parameters

estimator : scikit-learn estimator
The scikit-learn tree or tree ensemble estimator object.
data : ndarray
A data matrix that can be used to update the number of samples in each node of the tree(s) in the supplied estimator object. This affects the node values of the resulting engineered features.
Expand source code 
class TreeTransformer(BlockTransformerBase, ABC):
    """
    A block transformer that transforms data using a representation built from
    local decision stumps from a tree or tree ensemble. The transformer also
    comes with metadata on the local decision stumps and methods that allow for
    transformations using sub-representations corresponding to each of the
    original features.

    Parameters
    ----------
    estimator: scikit-learn estimator
        The scikit-learn tree or tree ensemble estimator object.
    data: ndarray
        A data matrix that can be used to update the number of samples in each
        node of the tree(s) in the supplied estimator object. This affects
        the node values of the resulting engineered features.
    """

    def __init__(self, estimator, data=None):
        super().__init__()
        self.estimator = estimator
        self.oob_seed = self.estimator.random_state
        # Check if single tree or tree ensemble
        if isinstance(estimator, BaseEnsemble):
            tree_models = estimator.estimators_
            if data is not None:
                # If a data matrix is supplied, use it to update the number
                # of samples in each node
                for tree_model in tree_models:
                    _update_n_node_samples(tree_model, data)
        else:
            tree_models = [estimator]
        # Make stumps for each tree
        all_stumps = []
        for tree_model in tree_models:
            tree_stumps = make_stumps(tree_model.tree_)
            all_stumps += tree_stumps
        # Identify the stumps that split on feature k, for each k
        self.stumps = defaultdict(list)
        for stump in all_stumps:
            self.stumps[stump.feature].append(stump)
        self.n_splits = {k: len(stumps) for k, stumps in self.stumps.items()}

    def _fit_one_feature(self, X, k):
        stump_features = tree_feature_transform(self.stumps[k], X)
        self._centers[k] = np.mean(stump_features, axis=0)
        self._scales[k] = np.std(stump_features, axis=0)

    def _transform_one_feature(self, X, k):
        return tree_feature_transform(self.stumps[k], X)

    def _fit_transform_one_feature(self, X, k):
        stump_features = tree_feature_transform(self.stumps[k], X)
        self._centers[k] = np.mean(stump_features, axis=0)
        self._scales[k] = np.std(stump_features, axis=0)
        return stump_features

Ancestors

Inherited members