75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
import quapy as qp
|
|
from quapy.protocol import UPP
|
|
from quapy.method.aggregative import DMy
|
|
from sklearn.linear_model import LogisticRegression
|
|
import numpy as np
|
|
from time import time
|
|
|
|
"""
|
|
In this example, we show how to perform model selection on a DistributionMatching quantifier.
|
|
"""
|
|
|
|
model = DMy()
|
|
|
|
qp.environ['SAMPLE_SIZE'] = 100
|
|
|
|
print(f'running model selection with N_JOBS={qp.environ["N_JOBS"]}; '
|
|
f'to increase the number of jobs use:\n> N_JOBS=-1 python3 1.model_selection.py\n'
|
|
f'alternatively, you can set this variable within the script as:\n'
|
|
f'import quapy as qp\n'
|
|
f'qp.environ["N_JOBS"]=-1')
|
|
|
|
training, test = qp.datasets.fetch_UCIMulticlassDataset('letter').train_test
|
|
|
|
with qp.util.temp_seed(0):
|
|
|
|
# The model will be returned by the fit method of GridSearchQ.
|
|
# Every combination of hyper-parameters will be evaluated by confronting the
|
|
# quantifier thus configured against a series of samples generated by means
|
|
# of a sample generation protocol. For this example, we will use the
|
|
# artificial-prevalence protocol (APP), that generates samples with prevalence
|
|
# values in the entire range of values from a grid (e.g., [0, 0.1, 0.2, ..., 1]).
|
|
# We devote 30% of the dataset for this exploration.
|
|
training, validation = training.split_stratified(train_prop=0.7)
|
|
protocol = UPP(validation)
|
|
|
|
# We will explore a classification-dependent hyper-parameter (e.g., the 'C'
|
|
# hyper-parameter of LogisticRegression) and a quantification-dependent hyper-parameter
|
|
# (e.g., the number of bins in a DistributionMatching quantifier).
|
|
# Classifier-dependent hyper-parameters have to be marked with a prefix "classifier__"
|
|
# in order to let the quantifier know this hyper-parameter belongs to its underlying
|
|
# classifier.
|
|
# We consider 7 values for the classifier and 7 values for the quantifier.
|
|
# QuaPy is optimized so that only 7 classifiers are trained, and then reused to test the
|
|
# different configurations of the quantifier. In other words, quapy avoids to train
|
|
# the classifier 7x7 times.
|
|
param_grid = {
|
|
'classifier__C': np.logspace(-3, 3, 7),
|
|
'nbins': [2, 3, 4, 5, 10, 15, 20]
|
|
}
|
|
|
|
tinit = time()
|
|
|
|
model = qp.model_selection.GridSearchQ(
|
|
model=model,
|
|
param_grid=param_grid,
|
|
protocol=protocol,
|
|
error='mae', # the error to optimize is the MAE (a quantification-oriented loss)
|
|
refit=False, # retrain on the whole labelled set once done
|
|
# raise_errors=False,
|
|
verbose=True # show information as the process goes on
|
|
).fit(training)
|
|
|
|
tend = time()
|
|
|
|
print(f'model selection ended: best hyper-parameters={model.best_params_}')
|
|
model = model.best_model_
|
|
|
|
# evaluation in terms of MAE
|
|
# we use the same evaluation protocol (APP) on the test set
|
|
mae_score = qp.evaluation.evaluate(model, protocol=UPP(test), error_metric='mae')
|
|
|
|
print(f'MAE={mae_score:.5f}')
|
|
print(f'model selection took {tend-tinit:.1f}s')
|
|
|