Skip to content

Commit

Permalink
ENH: V1 cqr implmentation (#576)
Browse files Browse the repository at this point in the history 
ENH: V1 cqr implmentation
  • Loading branch information
@Valentin-Laurent @jawadhussein462
Valentin-Laurent authored and jawadhussein462 committed Dec 20, 2024
1 parent a59740b commit 5346085
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 104 deletions.
6 changes: 4 additions & 2 deletions mapie_v1/_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,11 @@ def check_if_X_y_different_from_fit(

def make_intervals_single_if_single_alpha(
intervals: NDArray,
alphas: List[float]
alphas: Union[float, List[float]]
) -> NDArray:
if len(alphas) == 1:
if isinstance(alphas, float):
return intervals[:, :, 0]
if isinstance(alphas, list) and len(alphas) == 1:
return intervals[:, :, 0]
return intervals

Expand Down
173 changes: 80 additions & 93 deletions mapie_v1/integration_tests/tests/test_regression.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from __future__ import annotations
from typing import Optional, Union, Dict, Tuple, Type
from typing import Optional, Union, Dict, Type

import numpy as np
import pytest
Expand All @@ -9,7 +9,8 @@
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import QuantileRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import train_test_split

from mapie.subsample import Subsample
from mapie._typing import ArrayLike
Expand Down Expand Up @@ -109,16 +110,17 @@ def test_intervals_and_predictions_exact_equality_split(
"random_state": RANDOM_STATE,
}

v0, v1 = select_models_by_strategy(cv)
compare_model_predictions_and_intervals(model_v0=v0,
model_v1=v1,
X=X_split,
y=y_split,
v0_params=v0_params,
v1_params=v1_params,
test_size=test_size,
random_state=RANDOM_STATE,
prefit=prefit)
compare_model_predictions_and_intervals(
model_v0=MapieRegressorV0,
model_v1=SplitConformalRegressor,
X=X_split,
y=y_split,
v0_params=v0_params,
v1_params=v1_params,
test_size=test_size,
prefit=prefit,
random_state=RANDOM_STATE,
)


params_test_cases_cross = [
Expand Down Expand Up @@ -185,11 +187,16 @@ def test_intervals_and_predictions_exact_equality_split(

@pytest.mark.parametrize("params_cross", params_test_cases_cross)
def test_intervals_and_predictions_exact_equality_cross(params_cross):
v0_params = params_cross["v0"]
v1_params = params_cross["v1"]

v0, v1 = select_models_by_strategy("cross")
compare_model_predictions_and_intervals(v0, v1, X, y, v0_params, v1_params)
compare_model_predictions_and_intervals(
model_v0=MapieRegressorV0,
model_v1=CrossConformalRegressor,
X=X,
y=y,
v0_params=params_cross["v0"],
v1_params=params_cross["v1"],
random_state=RANDOM_STATE,
)


params_test_cases_jackknife = [
Expand Down Expand Up @@ -268,64 +275,78 @@ def test_intervals_and_predictions_exact_equality_cross(params_cross):
]


@pytest.mark.parametrize("params_jackknife", params_test_cases_jackknife)
def test_intervals_and_predictions_exact_equality_jackknife(params_jackknife):

compare_model_predictions_and_intervals(
model_v0=MapieRegressorV0,
model_v1=JackknifeAfterBootstrapRegressor,
X=X,
y=y,
v0_params=params_jackknife["v0"],
v1_params=params_jackknife["v1"],
random_state=RANDOM_STATE,
)


split_model = QuantileRegressor(
solver="highs-ds",
alpha=0.0,
)

lgbm_models = []
lgbm_alpha = 0.1
for alpha_ in [lgbm_alpha / 2, (1 - (lgbm_alpha / 2)), 0.5]:
estimator_ = LGBMRegressor(
objective='quantile',
gbr_models = []
gbr_alpha = 0.1

for alpha_ in [gbr_alpha / 2, (1 - (gbr_alpha / 2)), 0.5]:
estimator_ = GradientBoostingRegressor(
loss='quantile',
alpha=alpha_,
n_estimators=100,
learning_rate=0.1,
max_depth=3
)
lgbm_models.append(estimator_)


@pytest.mark.parametrize("params_jackknife", params_test_cases_jackknife)
def test_intervals_and_predictions_exact_equality_jackknife(params_jackknife):
v0_params = params_jackknife["v0"]
v1_params = params_jackknife["v1"]

v0, v1 = select_models_by_strategy("jackknife")
compare_model_predictions_and_intervals(v0, v1, X, y, v0_params, v1_params)
gbr_models.append(estimator_)

sample_weight_train = train_test_split(
X,
y,
sample_weight,
test_size=0.4,
random_state=RANDOM_STATE
)[-2]

params_test_cases_quantile = [
{
"v0": {
"alpha": 0.2,
"cv": "split",
"method": "quantile",
"calib_size": 0.3,
"calib_size": 0.4,
"sample_weight": sample_weight,
"random_state": RANDOM_STATE,
},
"v1": {
"confidence_level": 0.8,
"prefit": False,
"test_size": 0.3,
"fit_params": {"sample_weight": sample_weight},
"test_size": 0.4,
"fit_params": {"sample_weight": sample_weight_train},
"random_state": RANDOM_STATE,
},
},
{
"v0": {
"estimator": lgbm_models,
"alpha": lgbm_alpha,
"estimator": gbr_models,
"cv": "prefit",
"method": "quantile",
"calib_size": 0.3,
"calib_size": 0.2,
"sample_weight": sample_weight,
"optimize_beta": True,
"random_state": RANDOM_STATE,
},
"v1": {
"estimator": lgbm_models,
"confidence_level": 1-lgbm_alpha,
"estimator": gbr_models,
"prefit": True,
"test_size": 0.3,
"test_size": 0.2,
"fit_params": {"sample_weight": sample_weight},
"minimize_interval_width": True,
"random_state": RANDOM_STATE,
Expand Down Expand Up @@ -378,58 +399,17 @@ def test_intervals_and_predictions_exact_equality_quantile(params_quantile):
test_size = v1_params["test_size"] if "test_size" in v1_params else None
prefit = ("prefit" in v1_params) and v1_params["prefit"]

v0, v1 = select_models_by_strategy("quantile")
compare_model_predictions_and_intervals(model_v0=v0,
model_v1=v1,
X=X,
y=y,
v0_params=v0_params,
v1_params=v1_params,
test_size=test_size,
prefit=prefit,
random_state=RANDOM_STATE)


def select_models_by_strategy(
strategy_key: str
) -> Tuple[
Type[Union[MapieRegressorV0, MapieQuantileRegressorV0]],
Type[Union[
SplitConformalRegressor,
CrossConformalRegressor,
JackknifeAfterBootstrapRegressor,
ConformalizedQuantileRegressor
]]
]:

model_v0: Type[Union[MapieRegressorV0, MapieQuantileRegressorV0]]
model_v1: Type[Union[
SplitConformalRegressor,
CrossConformalRegressor,
JackknifeAfterBootstrapRegressor,
ConformalizedQuantileRegressor
]]

if strategy_key in ["split", "prefit"]:
model_v1 = SplitConformalRegressor
model_v0 = MapieRegressorV0

elif strategy_key == "cross":
model_v1 = CrossConformalRegressor
model_v0 = MapieRegressorV0

elif strategy_key == "jackknife":
model_v1 = JackknifeAfterBootstrapRegressor
model_v0 = MapieRegressorV0

elif strategy_key == "quantile":
model_v1 = ConformalizedQuantileRegressor
model_v0 = MapieQuantileRegressorV0

else:
raise ValueError(f"Unknown strategy key: {strategy_key}")

return model_v0, model_v1
compare_model_predictions_and_intervals(
model_v0=MapieQuantileRegressorV0,
model_v1=ConformalizedQuantileRegressor,
X=X,
y=y,
v0_params=v0_params,
v1_params=v1_params,
test_size=test_size,
prefit=prefit,
random_state=RANDOM_STATE,
)


def compare_model_predictions_and_intervals(
Expand All @@ -446,12 +426,16 @@ def compare_model_predictions_and_intervals(
v1_params: Dict = {},
prefit: bool = False,
test_size: Optional[float] = None,
sample_weight: Optional[ArrayLike] = None,
random_state: int = 42,
) -> None:

if test_size is not None:
X_train, X_conf, y_train, y_conf = train_test_split_shuffle(
X, y, test_size=test_size, random_state=random_state
X,
y,
test_size=test_size,
random_state=random_state,
)
else:
X_train, X_conf, y_train, y_conf = X, X, y, y
Expand Down Expand Up @@ -486,6 +470,9 @@ def compare_model_predictions_and_intervals(
v1.conformalize(X_conf, y_conf, **v1_conformalize_params)

v0_predict_params = filter_params(v0.predict, v0_params)
if 'alpha' in v0_init_params:
v0_predict_params.pop('alpha')

v1_predict_params = filter_params(v1.predict, v1_params)
v1_predict_set_params = filter_params(v1.predict_set, v1_params)

Expand Down
Loading

0 comments on commit 5346085

Please sign in to comment.