import quapy as qp
from quapy.protocol import UPP
from quapy.method.aggregative import DMy
from sklearn.linear_model import LogisticRegression
import numpy as np
from time import time

"""
In this example, we show how to perform model selection on a DistributionMatching quantifier.
"""

model = DMy()

qp.environ['SAMPLE_SIZE'] = 100

print(f'running model selection with N_JOBS={qp.environ["N_JOBS"]}; '
      f'to increase the number of jobs use:\n> N_JOBS=-1 python3 1.model_selection.py\n'
      f'alternatively, you can set this variable within the script as:\n'
      f'import quapy as qp\n'
      f'qp.environ["N_JOBS"]=-1')

training, test = qp.datasets.fetch_UCIMulticlassDataset('letter').train_test

with qp.util.temp_seed(0):

    # The model will be returned by the fit method of GridSearchQ.
    # Every combination of hyper-parameters will be evaluated by confronting the
    # quantifier thus configured against a series of samples generated by means
    # of a sample generation protocol. For this example, we will use the
    # artificial-prevalence protocol (APP), that generates samples with prevalence
    # values in the entire range of values from a grid (e.g., [0, 0.1, 0.2, ..., 1]).
    # We devote 30% of the dataset for this exploration.
    training, validation = training.split_stratified(train_prop=0.7)
    protocol = UPP(validation)

    # We will explore a classification-dependent hyper-parameter (e.g., the 'C'
    # hyper-parameter of LogisticRegression) and a quantification-dependent hyper-parameter
    # (e.g., the number of bins in a DistributionMatching quantifier).
    # Classifier-dependent hyper-parameters have to be marked with a prefix "classifier__"
    # in order to let the quantifier know this hyper-parameter belongs to its underlying
    # classifier.
    # We consider 7 values for the classifier and 7 values for the quantifier.
    # QuaPy is optimized so that only 7 classifiers are trained, and then reused to test the
    # different configurations of the quantifier. In other words, quapy avoids to train
    # the classifier 7x7 times.
    param_grid = {
        'classifier__C': np.logspace(-3, 3, 7),
        'nbins': [2, 3, 4, 5, 10, 15, 20]
    }

    tinit = time()

    model = qp.model_selection.GridSearchQ(
        model=model,
        param_grid=param_grid,
        protocol=protocol,
        error='mae',  # the error to optimize is the MAE (a quantification-oriented loss)
        refit=False,   # retrain on the whole labelled set once done
        # raise_errors=False,
        verbose=True  # show information as the process goes on
    ).fit(training)

tend = time()

print(f'model selection ended: best hyper-parameters={model.best_params_}')
model = model.best_model_

# evaluation in terms of MAE
# we use the same evaluation protocol (APP) on the test set
mae_score = qp.evaluation.evaluate(model, protocol=UPP(test), error_metric='mae')

print(f'MAE={mae_score:.5f}')
print(f'model selection took {tend-tinit:.1f}s')