forked from moreo/QuaPy
189 lines
7.9 KiB
Python
189 lines
7.9 KiB
Python
from sklearn.linear_model import LogisticRegression
|
|
from sklearn.model_selection import GridSearchCV
|
|
from sklearn.svm import LinearSVC, LinearSVR
|
|
import quapy as qp
|
|
import quapy.functional as F
|
|
import sys
|
|
import numpy as np
|
|
|
|
from NewMethods.methods import AveragePoolQuantification
|
|
from classification.methods import PCALR
|
|
from classification.neural import NeuralClassifierTrainer, CNNnet
|
|
from method.meta import EPACC
|
|
from quapy.model_selection import GridSearchQ
|
|
|
|
# dataset = qp.datasets.fetch_UCIDataset('sonar', verbose=True)
|
|
# sys.exit(0)
|
|
|
|
|
|
qp.environ['SAMPLE_SIZE'] = 500
|
|
#param_grid = {'C': np.logspace(-3,3,7), 'class_weight': ['balanced', None]}
|
|
param_grid = {'C': np.logspace(0,3,4), 'class_weight': ['balanced']}
|
|
max_evaluations = 500
|
|
|
|
sample_size = qp.environ['SAMPLE_SIZE']
|
|
binary = False
|
|
svmperf_home = './svm_perf_quantification'
|
|
|
|
if binary:
|
|
#dataset = qp.datasets.fetch_reviews('kindle', tfidf=True, min_df=5)
|
|
dataset = qp.datasets.fetch_UCIDataset('german', verbose=True)
|
|
#qp.data.preprocessing.index(dataset, inplace=True)
|
|
|
|
else:
|
|
dataset = qp.datasets.fetch_twitter('gasp', for_model_selection=True, min_df=5, pickle=True)
|
|
#dataset.training = dataset.training.sampling(sample_size, 0.2, 0.5, 0.3)
|
|
|
|
print(f'dataset loaded: #training={len(dataset.training)} #test={len(dataset.test)}')
|
|
|
|
|
|
# training a quantifier
|
|
# learner = LogisticRegression(max_iter=1000)
|
|
#model = qp.method.aggregative.ClassifyAndCount(learner)
|
|
# model = qp.method.aggregative.AdjustedClassifyAndCount(learner)
|
|
# model = qp.method.aggregative.ProbabilisticClassifyAndCount(learner)
|
|
# model = qp.method.aggregative.ProbabilisticAdjustedClassifyAndCount(learner)
|
|
# model = qp.method.aggregative.HellingerDistanceY(learner)
|
|
# model = qp.method.aggregative.ExpectationMaximizationQuantifier(learner)
|
|
# model = qp.method.aggregative.ExplicitLossMinimisationBinary(svmperf_home, loss='q', C=100)
|
|
# model = qp.method.aggregative.SVMQ(svmperf_home, C=1)
|
|
|
|
#learner = PCALR()
|
|
#learner = NeuralClassifierTrainer(CNNnet(dataset.vocabulary_size, dataset.n_classes))
|
|
#print(learner.get_params())
|
|
#model = qp.method.meta.QuaNet(learner, sample_size, device='cpu')
|
|
|
|
#learner = GridSearchCV(LogisticRegression(max_iter=1000), param_grid=param_grid, n_jobs=-1, verbose=1)
|
|
#learner = LogisticRegression(max_iter=1000)
|
|
# model = qp.method.aggregative.ClassifyAndCount(learner)
|
|
|
|
param_mod_sel = {
|
|
'sample_size': 100,
|
|
'n_prevpoints': 21,
|
|
'n_repetitions': 5,
|
|
'verbose': False
|
|
}
|
|
common = {
|
|
'max_sample_size': 50,
|
|
'n_jobs': -1,
|
|
'param_grid': {'C': np.logspace(0,2,2), 'class_weight': ['balanced']},
|
|
'param_mod_sel': param_mod_sel,
|
|
'val_split': 0.4,
|
|
'min_pos': 10,
|
|
'size':6,
|
|
'red_size':3
|
|
}
|
|
|
|
# hyperparameters will be evaluated within each quantifier of the ensemble, and so the typical model selection
|
|
# will be skipped (by setting hyperparameters to None)
|
|
model = EPACC(LogisticRegression(max_iter=100), optim='mrae', policy='mrae', **common)
|
|
|
|
"""
|
|
Problemas:
|
|
- La interfaz es muy fea, hay que conocer practicamente todos los detalles así que no ahorra nada con respecto a crear
|
|
un objeto con otros anidados dentro
|
|
- El fit genera las prevalences random, y esto hace que despues de la model selection, un nuevo fit tire todo el trabajo
|
|
hecho.
|
|
- El fit de un GridSearcQ tiene dentro un best_estimator, pero después de la model selection, hacer fit otra vez sobre
|
|
este objeto no se limita a re-entrenar el modelo con los mejores parámetros, sino que inicia una nueva búsqueda
|
|
en modo grid search.
|
|
- Posible solución (no vale): sería hacer directamente model selection con el benchmark final, aunque esto haría que los hyper-
|
|
parámetros se buscasen en un conjunto diferente del resto de models....
|
|
- Posible solución:
|
|
- Elegir las prevalences en init
|
|
-
|
|
- Problema: el parámetro val_split es muy ambiguo en todo el framework. Por ejemplo, en EPACC podría ser un float que,
|
|
en el caso de un GridSearchQ podría referir al split de validación para los hyperparámetros o al split que usa PACC
|
|
para encontrar los parámetros...
|
|
"""
|
|
|
|
# regressor = LinearSVR(max_iter=10000)
|
|
# param_grid = {'C': np.logspace(-1,3,5)}
|
|
# model = AveragePoolQuantification(regressor, sample_size, trials=5000, n_components=500, zscore=False)
|
|
|
|
# model = qp.method.meta.EHDy(learner, param_grid=param_grid, optim='mae',
|
|
# sample_size=sample_size, eval_budget=max_evaluations//10, n_jobs=-1)
|
|
#model = qp.method.aggregative.ClassifyAndCount(learner)
|
|
|
|
# model = qp.method.meta.QuaNet(PCALR(n_components=100, max_iter=1000),
|
|
# sample_size=100,
|
|
# patience=10,
|
|
# tr_iter_per_poch=500, va_iter_per_poch=100, #lstm_nlayers=2, lstm_hidden_size=64,
|
|
# ff_layers=[500, 250, 50],
|
|
# checkpointdir='./checkpoint', device='cuda')
|
|
|
|
if qp.isbinary(model) and not qp.isbinary(dataset):
|
|
model = qp.method.aggregative.OneVsAll(model)
|
|
|
|
|
|
# Model fit and Evaluation on the test data
|
|
# ----------------------------------------------------------------------------
|
|
|
|
print(f'fitting model {model.__class__.__name__}')
|
|
#train, val = dataset.training.split_stratified(0.6)
|
|
#model.fit(train, val_split=val)
|
|
qp.SAMPLE=1
|
|
qp.environ['SAMPLE_SIZE']=2
|
|
model.fit(dataset.training)
|
|
|
|
|
|
|
|
|
|
|
|
# estimating class prevalences
|
|
# print('quantifying')
|
|
# prevalences_estim = model.quantify(dataset.test.instances)
|
|
# prevalences_true = dataset.test.prevalence()
|
|
#
|
|
# evaluation (one single prediction)
|
|
# error = qp.error.mae(prevalences_true, prevalences_estim)
|
|
#
|
|
# print(f'Evaluation in test (1 eval)')
|
|
# print(f'true prevalence {F.strprev(prevalences_true)}')
|
|
# print(f'estim prevalence {F.strprev(prevalences_estim)}')
|
|
# print(f'mae={error:.3f}')
|
|
|
|
|
|
# Model fit and Evaluation according to the artificial sampling protocol
|
|
# ----------------------------------------------------------------------------
|
|
|
|
n_prevpoints = F.get_nprevpoints_approximation(combinations_budget=max_evaluations, n_classes=dataset.n_classes)
|
|
n_evaluations = F.num_prevalence_combinations(n_prevpoints, dataset.n_classes)
|
|
print(f'the prevalence interval [0,1] will be split in {n_prevpoints} prevalence points for each class, so that\n'
|
|
f'the requested maximum number of sample evaluations ({max_evaluations}) is not exceeded.\n'
|
|
f'For the {dataset.n_classes} classes this dataset has, this will yield a total of {n_evaluations} evaluations.')
|
|
|
|
true_prev, estim_prev = qp.evaluation.artificial_sampling_prediction(model, dataset.test, sample_size, n_prevpoints)
|
|
|
|
#qp.error.SAMPLE_SIZE = sample_size
|
|
print(f'Evaluation according to the artificial sampling protocol ({len(true_prev)} evals)')
|
|
for error in qp.error.QUANTIFICATION_ERROR:
|
|
score = error(true_prev, estim_prev)
|
|
print(f'{error.__name__}={score:.5f}')
|
|
|
|
sys.exit(0)
|
|
# Model selection and Evaluation according to the artificial sampling protocol
|
|
# ----------------------------------------------------------------------------
|
|
|
|
model_selection = GridSearchQ(model,
|
|
param_grid=param_grid,
|
|
sample_size=sample_size,
|
|
eval_budget=max_evaluations//10,
|
|
error='mae',
|
|
refit=True,
|
|
verbose=True,
|
|
timeout=60*60)
|
|
|
|
model = model_selection.fit(dataset.training, val_split=0.3)
|
|
#model = model_selection.fit(train, validation=val)
|
|
print(f'Model selection: best_params = {model_selection.best_params_}')
|
|
print(f'param scores:')
|
|
for params, score in model_selection.param_scores_.items():
|
|
print(f'\t{params}: {score:.5f}')
|
|
|
|
true_prev, estim_prev = qp.evaluation.artificial_sampling_prediction(model, dataset.test, sample_size, n_prevpoints)
|
|
|
|
print(f'After model selection: Evaluation according to the artificial sampling protocol ({len(true_prev)} evals)')
|
|
for error in qp.error.QUANTIFICATION_ERROR:
|
|
score = error(true_prev, estim_prev)
|
|
print(f'{error.__name__}={score:.5f}') |