cleaning master
This commit is contained in:
parent
daeb3cdd88
commit
9c926cc8a5
|
|
@ -1,48 +0,0 @@
|
|||
from sklearn.model_selection import GridSearchCV
|
||||
import numpy as np
|
||||
import quapy as qp
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
sample_size = 500
|
||||
qp.environ['SAMPLE_SIZE'] = sample_size
|
||||
|
||||
|
||||
|
||||
def gen_data():
|
||||
|
||||
data = qp.datasets.fetch_reviews('kindle', tfidf=True, min_df=5)
|
||||
|
||||
models = [
|
||||
qp.method.aggregative.CC,
|
||||
qp.method.aggregative.ACC,
|
||||
qp.method.aggregative.PCC,
|
||||
qp.method.aggregative.PACC,
|
||||
qp.method.aggregative.HDy,
|
||||
qp.method.aggregative.EMQ,
|
||||
qp.method.meta.ECC,
|
||||
qp.method.meta.EACC,
|
||||
qp.method.meta.EHDy,
|
||||
]
|
||||
|
||||
method_names, true_prevs, estim_prevs, tr_prevs = [], [], [], []
|
||||
for Quantifier in models:
|
||||
print(f'training {Quantifier.__name__}')
|
||||
lr = LogisticRegression(max_iter=1000, class_weight='balanced')
|
||||
# lr = GridSearchCV(lr, param_grid={'C':np.logspace(-3,3,7)}, n_jobs=-1)
|
||||
model = Quantifier(lr).fit(data.training)
|
||||
true_prev, estim_prev = qp.evaluation.artificial_sampling_prediction(
|
||||
model, data.test, sample_size, n_repetitions=20, n_prevpoints=11)
|
||||
|
||||
method_names.append(Quantifier.__name__)
|
||||
true_prevs.append(true_prev)
|
||||
estim_prevs.append(estim_prev)
|
||||
tr_prevs.append(data.training.prevalence())
|
||||
|
||||
return method_names, true_prevs, estim_prevs, tr_prevs
|
||||
|
||||
method_names, true_prevs, estim_prevs, tr_prevs = qp.util.pickled_resource('./plots/plot_data.pkl', gen_data)
|
||||
|
||||
qp.plot.error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=11, savepath='./plots/err_drift.png')
|
||||
qp.plot.binary_diagonal(method_names, true_prevs, estim_prevs, savepath='./plots/bin_diag.png')
|
||||
qp.plot.binary_bias_global(method_names, true_prevs, estim_prevs, savepath='./plots/bin_bias.png')
|
||||
qp.plot.binary_bias_bins(method_names, true_prevs, estim_prevs, nbins=11, savepath='./plots/bin_bias_bin.png')
|
||||
222
test.py
222
test.py
|
|
@ -1,222 +0,0 @@
|
|||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.svm import LinearSVC, LinearSVR
|
||||
import quapy as qp
|
||||
import quapy.functional as F
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
from NewMethods.methods import AveragePoolQuantification
|
||||
from classification.methods import PCALR
|
||||
from data import Dataset
|
||||
from method.meta import EPACC
|
||||
from quapy.model_selection import GridSearchQ
|
||||
from tqdm import tqdm
|
||||
import pandas as pd
|
||||
|
||||
sample_size=100
|
||||
qp.environ['SAMPLE_SIZE'] = sample_size
|
||||
|
||||
np.random.seed(0)
|
||||
|
||||
nfolds=5
|
||||
nrepeats=1
|
||||
|
||||
df = pd.DataFrame(columns=['dataset', 'method', 'mse'])
|
||||
for datasetname in qp.datasets.UCI_DATASETS:
|
||||
collection = qp.datasets.fetch_UCILabelledCollection(datasetname, verbose=False)
|
||||
scores = []
|
||||
pbar = tqdm(Dataset.kFCV(collection, nfolds=nfolds, nrepeats=nrepeats), total=nfolds*nrepeats)
|
||||
for data in pbar:
|
||||
pbar.set_description(f'{data.name}')
|
||||
# learner = GridSearchCV(LogisticRegression(class_weight='balanced'), param_grid={'C': np.logspace(-3,3,7)}, n_jobs=-1)
|
||||
learner = LogisticRegression(class_weight='balanced')
|
||||
# model = qp.method.aggregative.CC(learner)
|
||||
model = qp.method.meta.EHDy(learner, size=30, red_size=15, verbose=False)
|
||||
model.fit(data.training)
|
||||
err = qp.evaluation.artificial_sampling_eval(model, data.test, sample_size, n_prevpoints=101, n_jobs=-1,
|
||||
error_metric='mse', verbose=False)
|
||||
scores.append(err)
|
||||
|
||||
score = np.mean(scores)
|
||||
df = df.append({
|
||||
'dataset': datasetname,
|
||||
'method': model.__class__.__name__,
|
||||
'mse': score
|
||||
}, ignore_index=True)
|
||||
print(df)
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
||||
#param_grid = {'C': np.logspace(-3,3,7), 'class_weight': ['balanced', None]}
|
||||
param_grid = {'C': np.logspace(0,3,4), 'class_weight': ['balanced']}
|
||||
max_evaluations = 500
|
||||
|
||||
sample_size = qp.environ['SAMPLE_SIZE']
|
||||
binary = False
|
||||
svmperf_home = './svm_perf_quantification'
|
||||
|
||||
if binary:
|
||||
#dataset = qp.datasets.fetch_reviews('kindle', tfidf=True, min_df=5)
|
||||
dataset = qp.datasets.fetch_UCIDataset('german', verbose=True)
|
||||
#qp.data.preprocessing.index(dataset, inplace=True)
|
||||
|
||||
else:
|
||||
dataset = qp.datasets.fetch_twitter('gasp', for_model_selection=True, min_df=5, pickle=True)
|
||||
#dataset.training = dataset.training.sampling(sample_size, 0.2, 0.5, 0.3)
|
||||
|
||||
print(f'dataset loaded: #training={len(dataset.training)} #test={len(dataset.test)}')
|
||||
|
||||
|
||||
# training a quantifier
|
||||
# learner = LogisticRegression(max_iter=1000)
|
||||
#model = qp.method.aggregative.ClassifyAndCount(learner)
|
||||
# model = qp.method.aggregative.AdjustedClassifyAndCount(learner)
|
||||
# model = qp.method.aggregative.ProbabilisticClassifyAndCount(learner)
|
||||
# model = qp.method.aggregative.ProbabilisticAdjustedClassifyAndCount(learner)
|
||||
# model = qp.method.aggregative.HellingerDistanceY(learner)
|
||||
# model = qp.method.aggregative.ExpectationMaximizationQuantifier(learner)
|
||||
# model = qp.method.aggregative.ExplicitLossMinimisationBinary(svmperf_home, loss='q', C=100)
|
||||
# model = qp.method.aggregative.SVMQ(svmperf_home, C=1)
|
||||
|
||||
#learner = PCALR()
|
||||
#learner = NeuralClassifierTrainer(CNNnet(dataset.vocabulary_size, dataset.n_classes))
|
||||
#print(learner.get_params())
|
||||
#model = qp.method.meta.QuaNet(learner, sample_size, device='cpu')
|
||||
|
||||
#learner = GridSearchCV(LogisticRegression(max_iter=1000), param_grid=param_grid, n_jobs=-1, verbose=1)
|
||||
#learner = LogisticRegression(max_iter=1000)
|
||||
# model = qp.method.aggregative.ClassifyAndCount(learner)
|
||||
|
||||
param_mod_sel = {
|
||||
'sample_size': 100,
|
||||
'n_prevpoints': 21,
|
||||
'n_repetitions': 5,
|
||||
'verbose': False
|
||||
}
|
||||
common = {
|
||||
'max_sample_size': 50,
|
||||
'n_jobs': -1,
|
||||
'param_grid': {'C': np.logspace(0,2,2), 'class_weight': ['balanced']},
|
||||
'param_mod_sel': param_mod_sel,
|
||||
'val_split': 0.4,
|
||||
'min_pos': 10,
|
||||
'size':6,
|
||||
'red_size':3
|
||||
}
|
||||
|
||||
# hyperparameters will be evaluated within each quantifier of the ensemble, and so the typical model selection
|
||||
# will be skipped (by setting hyperparameters to None)
|
||||
model = EPACC(LogisticRegression(max_iter=100), optim='mrae', policy='mrae', **common)
|
||||
|
||||
"""
|
||||
Problemas:
|
||||
- La interfaz es muy fea, hay que conocer practicamente todos los detalles así que no ahorra nada con respecto a crear
|
||||
un objeto con otros anidados dentro
|
||||
- El fit genera las prevalences random, y esto hace que despues de la model selection, un nuevo fit tire todo el trabajo
|
||||
hecho.
|
||||
- El fit de un GridSearcQ tiene dentro un best_estimator, pero después de la model selection, hacer fit otra vez sobre
|
||||
este objeto no se limita a re-entrenar el modelo con los mejores parámetros, sino que inicia una nueva búsqueda
|
||||
en modo grid search.
|
||||
- Posible solución (no vale): sería hacer directamente model selection con el benchmark final, aunque esto haría que los hyper-
|
||||
parámetros se buscasen en un conjunto diferente del resto de models....
|
||||
- Posible solución:
|
||||
- Elegir las prevalences en init
|
||||
-
|
||||
- Problema: el parámetro val_split es muy ambiguo en todo el framework. Por ejemplo, en EPACC podría ser un float que,
|
||||
en el caso de un GridSearchQ podría referir al split de validación para los hyperparámetros o al split que usa PACC
|
||||
para encontrar los parámetros...
|
||||
"""
|
||||
|
||||
# regressor = LinearSVR(max_iter=10000)
|
||||
# param_grid = {'C': np.logspace(-1,3,5)}
|
||||
# model = AveragePoolQuantification(regressor, sample_size, trials=5000, n_components=500, zscore=False)
|
||||
|
||||
# model = qp.method.meta.EHDy(learner, param_grid=param_grid, optim='mae',
|
||||
# sample_size=sample_size, eval_budget=max_evaluations//10, n_jobs=-1)
|
||||
#model = qp.method.aggregative.ClassifyAndCount(learner)
|
||||
|
||||
# model = qp.method.meta.QuaNet(PCALR(n_components=100, max_iter=1000),
|
||||
# sample_size=100,
|
||||
# patience=10,
|
||||
# tr_iter_per_poch=500, va_iter_per_poch=100, #lstm_nlayers=2, lstm_hidden_size=64,
|
||||
# ff_layers=[500, 250, 50],
|
||||
# checkpointdir='./checkpoint', device='cuda')
|
||||
|
||||
if qp.isbinary(model) and not qp.isbinary(dataset):
|
||||
model = qp.method.aggregative.OneVsAll(model)
|
||||
|
||||
|
||||
# Model fit and Evaluation on the test data
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
print(f'fitting model {model.__class__.__name__}')
|
||||
#train, val = dataset.training.split_stratified(0.6)
|
||||
#model.fit(train, val_split=val)
|
||||
qp.SAMPLE=1
|
||||
qp.environ['SAMPLE_SIZE']=2
|
||||
model.fit(dataset.training)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# estimating class prevalences
|
||||
# print('quantifying')
|
||||
# prevalences_estim = model.quantify(dataset.test.instances)
|
||||
# prevalences_true = dataset.test.prevalence()
|
||||
#
|
||||
# evaluation (one single prediction)
|
||||
# error = qp.error.mae(prevalences_true, prevalences_estim)
|
||||
#
|
||||
# print(f'Evaluation in test (1 eval)')
|
||||
# print(f'true prevalence {F.strprev(prevalences_true)}')
|
||||
# print(f'estim prevalence {F.strprev(prevalences_estim)}')
|
||||
# print(f'mae={error:.3f}')
|
||||
|
||||
|
||||
# Model fit and Evaluation according to the artificial sampling protocol
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
n_prevpoints = F.get_nprevpoints_approximation(combinations_budget=max_evaluations, n_classes=dataset.n_classes)
|
||||
n_evaluations = F.num_prevalence_combinations(n_prevpoints, dataset.n_classes)
|
||||
print(f'the prevalence interval [0,1] will be split in {n_prevpoints} prevalence points for each class, so that\n'
|
||||
f'the requested maximum number of sample evaluations ({max_evaluations}) is not exceeded.\n'
|
||||
f'For the {dataset.n_classes} classes this dataset has, this will yield a total of {n_evaluations} evaluations.')
|
||||
|
||||
true_prev, estim_prev = qp.evaluation.artificial_sampling_prediction(model, dataset.test, sample_size, n_prevpoints)
|
||||
|
||||
#qp.error.SAMPLE_SIZE = sample_size
|
||||
print(f'Evaluation according to the artificial sampling protocol ({len(true_prev)} evals)')
|
||||
for error in qp.error.QUANTIFICATION_ERROR:
|
||||
score = error(true_prev, estim_prev)
|
||||
print(f'{error.__name__}={score:.5f}')
|
||||
|
||||
sys.exit(0)
|
||||
# Model selection and Evaluation according to the artificial sampling protocol
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
model_selection = GridSearchQ(model,
|
||||
param_grid=param_grid,
|
||||
sample_size=sample_size,
|
||||
eval_budget=max_evaluations//10,
|
||||
error='mae',
|
||||
refit=True,
|
||||
verbose=True,
|
||||
timeout=60*60)
|
||||
|
||||
model = model_selection.fit(dataset.training, val_split=0.3)
|
||||
#model = model_selection.fit(train, validation=val)
|
||||
print(f'Model selection: best_params = {model_selection.best_params_}')
|
||||
print(f'param scores:')
|
||||
for params, score in model_selection.param_scores_.items():
|
||||
print(f'\t{params}: {score:.5f}')
|
||||
|
||||
true_prev, estim_prev = qp.evaluation.artificial_sampling_prediction(model, dataset.test, sample_size, n_prevpoints)
|
||||
|
||||
print(f'After model selection: Evaluation according to the artificial sampling protocol ({len(true_prev)} evals)')
|
||||
for error in qp.error.QUANTIFICATION_ERROR:
|
||||
score = error(true_prev, estim_prev)
|
||||
print(f'{error.__name__}={score:.5f}')
|
||||
Loading…
Reference in New Issue