Wrapper for sparse, integer linear models.
minimizes norm(X * w - y, 2) + alpha * norm(w, 1)
with integer coefficients in w
Requires installation of a solver for mixed-integer linear programs, e.g. gurobi, mosek, or cplex
Expand source code
'''
Wrapper for sparse, integer linear models.
minimizes norm(X * w - y, 2) + alpha * norm(w, 1)
with integer coefficients in w
Requires installation of a solver for mixed-integer linear programs, e.g. gurobi, mosek, or cplex
'''
from imodels.util.introspection import TextMixin
from imodels.util.introspection import TextMixin
import warnings
import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
from sklearn.linear_model import LinearRegression, Lasso, LogisticRegression
from sklearn.utils.multiclass import check_classification_targets
from sklearn.utils.validation import check_X_y, check_array, check_is_fitted
from imodels.util.arguments import check_predict_X, set_feature_names_in
from imodels.util.optional_deps import warn_optional_dependency
def _warn_if_rounding_collapsed(coef, rounded):
"""Warn when rounding to integers has destroyed the fitted model.
SLIM's coefficients are integers, so a feature whose natural coefficient is
much smaller than 1 rounds to zero. On data whose columns are on very
different scales that silently removes most of the model: on breast_cancer,
raw features give a below-chance classifier while standardized ones reach
0.99 AUC.
"""
nonzero_before = int(np.count_nonzero(coef))
nonzero_after = int(np.count_nonzero(rounded))
if nonzero_before and nonzero_after * 2 < nonzero_before:
warnings.warn(
f"Rounding coefficients to integers left {nonzero_after} of "
f"{nonzero_before} nonzero, so most of the fitted model was lost. "
"SLIM uses integer coefficients, so features on very different "
"scales collapse when rounded; scale X first (for example with "
"sklearn.preprocessing.StandardScaler)."
)
class SLIMRegressor(TextMixin, RegressorMixin, BaseEstimator):
'''Sparse integer linear model
Parameters
----------
alpha: float
weight for sparsity penalty
'''
def __init__(self, alpha=0.01):
warn_optional_dependency(
'cvxpy', 'SLIMRegressor', fallback='rounding the coefficients of a non-integer fit',
purpose='used to solve the integer program exactly')
self.alpha = alpha
def fit(self, X, y, sample_weight=None):
'''fit a linear model with integer coefficient and L1 regularization.
In case the optimization fails, fit lasso and round coefs.
Parameters
----------
_sample_weight: np.ndarray (n,), optional
weight for each individual sample
'''
set_feature_names_in(self, X)
X, y = check_X_y(X, y)
self.n_features_in_ = X.shape[1]
self.model_ = LinearRegression()
try:
import cvxpy as cp # package for optimization, import here to make it optional
from cvxpy.error import SolverError
# declare the integer-valued optimization variable
w = cp.Variable(X.shape[1], integer=True)
# set up the minimization problem
residuals = X @ w - y
if sample_weight is not None:
residuals = cp.multiply(sample_weight, residuals)
mse = cp.sum_squares(residuals)
l1_penalty = self.alpha * cp.norm(w, 1)
obj = cp.Minimize(mse + l1_penalty)
prob = cp.Problem(obj)
try:
# solve the problem using an appropriate solver
prob.solve()
self.model_.coef_ = w.value.astype(int)
self.model_.intercept_ = 0
except SolverError:
warnings.warn("gurobi, mosek, or cplex solver required for mixed-integer "
"quadratic programming. Rounding non-integer coefficients instead.")
self._fit_backup(X, y, sample_weight)
except ImportError:
warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer "
"coefficients instead.")
self._fit_backup(X, y, sample_weight)
return self
def _fit_backup(self, X, y, sample_weight):
m = Lasso(alpha=self.alpha)
m.fit(X, y, sample_weight=sample_weight)
rounded = np.round(m.coef_).astype(int)
_warn_if_rounding_collapsed(m.coef_, rounded)
self.model_.coef_ = rounded
self.model_.intercept_ = m.intercept_
def predict(self, X):
check_is_fitted(self)
check_predict_X(self, X)
X = check_array(X)
return self.model_.predict(X)
class SLIMClassifier(TextMixin, ClassifierMixin, BaseEstimator):
def __init__(self, alpha=1):
'''Model is initialized during fitting
Parameters
----------
alpha: float
weight for sparsity penalty
'''
warn_optional_dependency(
'cvxpy', 'SLIMClassifier', fallback='rounding the coefficients of a non-integer fit',
purpose='used to solve the integer program exactly')
self.alpha = alpha
def fit(self, X, y, sample_weight=None):
'''fit a logistic model with integer coefficient and L1 regularization.
In case the optimization fails, fit lasso and round coefs.
Parameters
----------
_sample_weight: np.ndarray (n,), optional
weight for each individual sample
'''
set_feature_names_in(self, X)
X, y = check_X_y(X, y)
check_classification_targets(y)
self.n_features_in_ = X.shape[1]
self.classes_, y = np.unique(y, return_inverse=True) # deals with str inputs
self.model_ = LogisticRegression()
self.model_.classes_ = self.classes_
try:
import cvxpy as cp # package for optimization, import here to make it optional
from cvxpy.error import SolverError
# declare the integer-valued optimization variable
w = cp.Variable(X.shape[1], integer=True)
# set up the minimization problem
logits = -X @ w
residuals = cp.multiply(1 - y, logits) - cp.logistic(logits)
if sample_weight is not None:
residuals = cp.multiply(sample_weight, residuals)
celoss = -cp.sum(residuals)
l1_penalty = self.alpha * cp.norm(w, 1)
obj = cp.Minimize(celoss + l1_penalty)
prob = cp.Problem(obj)
try:
# solve the problem using an appropriate solver
prob.solve()
self.model_.coef_ = np.array([w.value.astype(int)])
self.model_.intercept_ = 0
except SolverError:
warnings.warn("mosek solver required for mixed-integer exponential cone "
"programming. Rounding non-integer coefficients instead")
self._fit_backup(X, y, sample_weight)
except ImportError:
warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer "
"coefficients instead.")
self._fit_backup(X, y, sample_weight)
return self
def _fit_backup(self, X, y, sample_weight=None):
m = LogisticRegression(C=1 / self.alpha)
m.fit(X, y, sample_weight=sample_weight)
rounded = np.round(m.coef_).astype(int)
_warn_if_rounding_collapsed(m.coef_, rounded)
self.model_.coef_ = rounded
self.model_.intercept_ = m.intercept_
def predict(self, X):
check_is_fitted(self)
check_predict_X(self, X)
X = check_array(X)
return self.model_.predict(X)
def predict_proba(self, X):
check_is_fitted(self)
check_predict_X(self, X)
X = check_array(X)
return self.model_.predict_proba(X)Classes
class SLIMClassifier (alpha=1)-
Prints a fitted model as readable text (see imodels.viz.text): its tree, rules, points or shape functions, with the numbers that make each prediction. Unfitted models, and models imodels.viz cannot read, print their parameters as before.
Model is initialized during fitting
Parameters
alpha:float- weight for sparsity penalty
Expand source code
class SLIMClassifier(TextMixin, ClassifierMixin, BaseEstimator): def __init__(self, alpha=1): '''Model is initialized during fitting Parameters ---------- alpha: float weight for sparsity penalty ''' warn_optional_dependency( 'cvxpy', 'SLIMClassifier', fallback='rounding the coefficients of a non-integer fit', purpose='used to solve the integer program exactly') self.alpha = alpha def fit(self, X, y, sample_weight=None): '''fit a logistic model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs. Parameters ---------- _sample_weight: np.ndarray (n,), optional weight for each individual sample ''' set_feature_names_in(self, X) X, y = check_X_y(X, y) check_classification_targets(y) self.n_features_in_ = X.shape[1] self.classes_, y = np.unique(y, return_inverse=True) # deals with str inputs self.model_ = LogisticRegression() self.model_.classes_ = self.classes_ try: import cvxpy as cp # package for optimization, import here to make it optional from cvxpy.error import SolverError # declare the integer-valued optimization variable w = cp.Variable(X.shape[1], integer=True) # set up the minimization problem logits = -X @ w residuals = cp.multiply(1 - y, logits) - cp.logistic(logits) if sample_weight is not None: residuals = cp.multiply(sample_weight, residuals) celoss = -cp.sum(residuals) l1_penalty = self.alpha * cp.norm(w, 1) obj = cp.Minimize(celoss + l1_penalty) prob = cp.Problem(obj) try: # solve the problem using an appropriate solver prob.solve() self.model_.coef_ = np.array([w.value.astype(int)]) self.model_.intercept_ = 0 except SolverError: warnings.warn("mosek solver required for mixed-integer exponential cone " "programming. Rounding non-integer coefficients instead") self._fit_backup(X, y, sample_weight) except ImportError: warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer " "coefficients instead.") self._fit_backup(X, y, sample_weight) return self def _fit_backup(self, X, y, sample_weight=None): m = LogisticRegression(C=1 / self.alpha) m.fit(X, y, sample_weight=sample_weight) rounded = np.round(m.coef_).astype(int) _warn_if_rounding_collapsed(m.coef_, rounded) self.model_.coef_ = rounded self.model_.intercept_ = m.intercept_ def predict(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict(X) def predict_proba(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict_proba(X)Ancestors
- TextMixin
- sklearn.base.ClassifierMixin
- sklearn.base.BaseEstimator
- sklearn.utils._estimator_html_repr._HTMLDocumentationLinkMixin
- sklearn.utils._metadata_requests._MetadataRequester
Methods
def fit(self, X, y, sample_weight=None)-
fit a logistic model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs.
Parameters
_sample_weight:np.ndarray (n,), optional- weight for each individual sample
Expand source code
def fit(self, X, y, sample_weight=None): '''fit a logistic model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs. Parameters ---------- _sample_weight: np.ndarray (n,), optional weight for each individual sample ''' set_feature_names_in(self, X) X, y = check_X_y(X, y) check_classification_targets(y) self.n_features_in_ = X.shape[1] self.classes_, y = np.unique(y, return_inverse=True) # deals with str inputs self.model_ = LogisticRegression() self.model_.classes_ = self.classes_ try: import cvxpy as cp # package for optimization, import here to make it optional from cvxpy.error import SolverError # declare the integer-valued optimization variable w = cp.Variable(X.shape[1], integer=True) # set up the minimization problem logits = -X @ w residuals = cp.multiply(1 - y, logits) - cp.logistic(logits) if sample_weight is not None: residuals = cp.multiply(sample_weight, residuals) celoss = -cp.sum(residuals) l1_penalty = self.alpha * cp.norm(w, 1) obj = cp.Minimize(celoss + l1_penalty) prob = cp.Problem(obj) try: # solve the problem using an appropriate solver prob.solve() self.model_.coef_ = np.array([w.value.astype(int)]) self.model_.intercept_ = 0 except SolverError: warnings.warn("mosek solver required for mixed-integer exponential cone " "programming. Rounding non-integer coefficients instead") self._fit_backup(X, y, sample_weight) except ImportError: warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer " "coefficients instead.") self._fit_backup(X, y, sample_weight) return self def predict(self, X)-
Expand source code
def predict(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict(X) def predict_proba(self, X)-
Expand source code
def predict_proba(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict_proba(X)
class SLIMRegressor (alpha=0.01)-
Sparse integer linear model Parameters
alpha:float- weight for sparsity penalty
Expand source code
class SLIMRegressor(TextMixin, RegressorMixin, BaseEstimator): '''Sparse integer linear model Parameters ---------- alpha: float weight for sparsity penalty ''' def __init__(self, alpha=0.01): warn_optional_dependency( 'cvxpy', 'SLIMRegressor', fallback='rounding the coefficients of a non-integer fit', purpose='used to solve the integer program exactly') self.alpha = alpha def fit(self, X, y, sample_weight=None): '''fit a linear model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs. Parameters ---------- _sample_weight: np.ndarray (n,), optional weight for each individual sample ''' set_feature_names_in(self, X) X, y = check_X_y(X, y) self.n_features_in_ = X.shape[1] self.model_ = LinearRegression() try: import cvxpy as cp # package for optimization, import here to make it optional from cvxpy.error import SolverError # declare the integer-valued optimization variable w = cp.Variable(X.shape[1], integer=True) # set up the minimization problem residuals = X @ w - y if sample_weight is not None: residuals = cp.multiply(sample_weight, residuals) mse = cp.sum_squares(residuals) l1_penalty = self.alpha * cp.norm(w, 1) obj = cp.Minimize(mse + l1_penalty) prob = cp.Problem(obj) try: # solve the problem using an appropriate solver prob.solve() self.model_.coef_ = w.value.astype(int) self.model_.intercept_ = 0 except SolverError: warnings.warn("gurobi, mosek, or cplex solver required for mixed-integer " "quadratic programming. Rounding non-integer coefficients instead.") self._fit_backup(X, y, sample_weight) except ImportError: warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer " "coefficients instead.") self._fit_backup(X, y, sample_weight) return self def _fit_backup(self, X, y, sample_weight): m = Lasso(alpha=self.alpha) m.fit(X, y, sample_weight=sample_weight) rounded = np.round(m.coef_).astype(int) _warn_if_rounding_collapsed(m.coef_, rounded) self.model_.coef_ = rounded self.model_.intercept_ = m.intercept_ def predict(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict(X)Ancestors
- TextMixin
- sklearn.base.RegressorMixin
- sklearn.base.BaseEstimator
- sklearn.utils._estimator_html_repr._HTMLDocumentationLinkMixin
- sklearn.utils._metadata_requests._MetadataRequester
Methods
def fit(self, X, y, sample_weight=None)-
fit a linear model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs.
Parameters
_sample_weight:np.ndarray (n,), optional- weight for each individual sample
Expand source code
def fit(self, X, y, sample_weight=None): '''fit a linear model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs. Parameters ---------- _sample_weight: np.ndarray (n,), optional weight for each individual sample ''' set_feature_names_in(self, X) X, y = check_X_y(X, y) self.n_features_in_ = X.shape[1] self.model_ = LinearRegression() try: import cvxpy as cp # package for optimization, import here to make it optional from cvxpy.error import SolverError # declare the integer-valued optimization variable w = cp.Variable(X.shape[1], integer=True) # set up the minimization problem residuals = X @ w - y if sample_weight is not None: residuals = cp.multiply(sample_weight, residuals) mse = cp.sum_squares(residuals) l1_penalty = self.alpha * cp.norm(w, 1) obj = cp.Minimize(mse + l1_penalty) prob = cp.Problem(obj) try: # solve the problem using an appropriate solver prob.solve() self.model_.coef_ = w.value.astype(int) self.model_.intercept_ = 0 except SolverError: warnings.warn("gurobi, mosek, or cplex solver required for mixed-integer " "quadratic programming. Rounding non-integer coefficients instead.") self._fit_backup(X, y, sample_weight) except ImportError: warnings.warn("Should install cvxpy with pip install cvxpy. Rounding non-integer " "coefficients instead.") self._fit_backup(X, y, sample_weight) return self def predict(self, X)-
Expand source code
def predict(self, X): check_is_fitted(self) check_predict_X(self, X) X = check_array(X) return self.model_.predict(X)