forked from moreo/QuaPy
cleaning
This commit is contained in:
parent
168c109794
commit
1aafd10e25
|
|
@ -1,174 +0,0 @@
|
||||||
import numpy as np
|
|
||||||
from sklearn.base import BaseEstimator
|
|
||||||
from sklearn.decomposition import PCA
|
|
||||||
from sklearn.preprocessing import StandardScaler
|
|
||||||
|
|
||||||
import quapy as qp
|
|
||||||
from typing import Union
|
|
||||||
|
|
||||||
from quapy.data import LabelledCollection
|
|
||||||
from quapy.method.base import BaseQuantifier, BinaryQuantifier
|
|
||||||
from quapy.method.aggregative import PACC, EMQ, HDy
|
|
||||||
import quapy.functional as F
|
|
||||||
from tqdm import tqdm
|
|
||||||
from scipy.sparse import issparse, csr_matrix
|
|
||||||
import scipy
|
|
||||||
|
|
||||||
|
|
||||||
class PACCSLD(PACC):
|
|
||||||
"""
|
|
||||||
This method combines the EMQ improved posterior probabilities with PACC.
|
|
||||||
Note: the posterior probabilities are re-calibrated with EMQ only during prediction, and not also during fit since,
|
|
||||||
for PACC, the validation split is known to have the same prevalence as the training set (this is because the split
|
|
||||||
is stratified) and thus the posterior probabilities should not be re-calibrated for a different prior (it actually
|
|
||||||
happens to degrades performance).
|
|
||||||
"""
|
|
||||||
|
|
||||||
def fit(self, data: qp.data.LabelledCollection, fit_learner=True, val_split:Union[float, int, qp.data.LabelledCollection]=0.4):
|
|
||||||
self.train_prevalence = F.prevalence_from_labels(data.labels, data.n_classes)
|
|
||||||
return super(PACCSLD, self).fit(data, fit_learner, val_split)
|
|
||||||
|
|
||||||
def aggregate(self, classif_posteriors):
|
|
||||||
priors, posteriors = EMQ.EM(self.train_prevalence, classif_posteriors, epsilon=1e-4)
|
|
||||||
return super(PACCSLD, self).aggregate(posteriors)
|
|
||||||
|
|
||||||
|
|
||||||
class HDySLD(HDy):
|
|
||||||
"""
|
|
||||||
This method combines the EMQ improved posterior probabilities with HDy.
|
|
||||||
Note: [same as PACCSLD]
|
|
||||||
"""
|
|
||||||
def fit(self, data: qp.data.LabelledCollection, fit_learner=True,
|
|
||||||
val_split: Union[float, int, qp.data.LabelledCollection] = 0.4):
|
|
||||||
self.train_prevalence = F.prevalence_from_labels(data.labels, data.n_classes)
|
|
||||||
return super(HDySLD, self).fit(data, fit_learner, val_split)
|
|
||||||
|
|
||||||
def aggregate(self, classif_posteriors):
|
|
||||||
priors, posteriors = EMQ.EM(self.train_prevalence, classif_posteriors, epsilon=1e-4)
|
|
||||||
return super(HDySLD, self).aggregate(posteriors)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class AveragePoolQuantification(BinaryQuantifier):
|
|
||||||
def __init__(self, learner, sample_size, trials, n_components=-1, zscore=False):
|
|
||||||
self.learner = learner
|
|
||||||
self.sample_size = sample_size
|
|
||||||
self.trials = trials
|
|
||||||
|
|
||||||
self.do_zscore = zscore
|
|
||||||
self.zscore = StandardScaler() if self.do_zscore else None
|
|
||||||
|
|
||||||
self.do_pca = n_components>0
|
|
||||||
self.pca = PCA(n_components) if self.do_pca else None
|
|
||||||
|
|
||||||
def fit(self, data: LabelledCollection):
|
|
||||||
training, validation = data.split_stratified(train_prop=0.7)
|
|
||||||
|
|
||||||
X, y = [], []
|
|
||||||
|
|
||||||
nprevpoints = F.get_nprevpoints_approximation(self.trials, data.n_classes)
|
|
||||||
for sample in tqdm(
|
|
||||||
training.artificial_sampling_generator(self.sample_size, n_prevalences=nprevpoints, repeats=1),
|
|
||||||
desc='generating averages'
|
|
||||||
):
|
|
||||||
X.append(sample.instances.mean(axis=0))
|
|
||||||
y.append(sample.prevalence()[1])
|
|
||||||
while len(X) < self.trials:
|
|
||||||
sample = training.sampling(self.sample_size, F.uniform_simplex_sampling(data.n_classes))
|
|
||||||
X.append(sample.instances.mean(axis=0))
|
|
||||||
y.append(sample.prevalence())
|
|
||||||
X = np.asarray(np.vstack(X))
|
|
||||||
y = np.asarray(y)
|
|
||||||
|
|
||||||
if self.do_pca:
|
|
||||||
X = self.pca.fit_transform(X)
|
|
||||||
print(X.shape)
|
|
||||||
|
|
||||||
if self.do_zscore:
|
|
||||||
X = self.zscore.fit_transform(X)
|
|
||||||
|
|
||||||
print('training regressor...')
|
|
||||||
self.regressor = self.learner.fit(X, y)
|
|
||||||
|
|
||||||
# correction at 0:
|
|
||||||
print('getting corrections...')
|
|
||||||
X0 = np.asarray(np.vstack([validation.sampling(self.sample_size, 0., shuffle=False).instances.mean(axis=0) for _ in range(100)]))
|
|
||||||
X1 = np.asarray(np.vstack([validation.sampling(self.sample_size, 1., shuffle=False).instances.mean(axis=0) for _ in range(100)]))
|
|
||||||
|
|
||||||
if self.do_pca:
|
|
||||||
X0 = self.pca.transform(X0)
|
|
||||||
X1 = self.pca.transform(X1)
|
|
||||||
|
|
||||||
if self.do_zscore:
|
|
||||||
X0 = self.zscore.transform(X0)
|
|
||||||
X1 = self.zscore.transform(X1)
|
|
||||||
|
|
||||||
self.correction_0 = self.regressor.predict(X0).mean()
|
|
||||||
self.correction_1 = self.regressor.predict(X1).mean()
|
|
||||||
|
|
||||||
print('correction-0', self.correction_0)
|
|
||||||
print('correction-1', self.correction_1)
|
|
||||||
print('done')
|
|
||||||
|
|
||||||
def quantify(self, instances):
|
|
||||||
ave = np.asarray(instances.mean(axis=0))
|
|
||||||
|
|
||||||
if self.do_pca:
|
|
||||||
ave = self.pca.transform(ave)
|
|
||||||
if self.do_zscore:
|
|
||||||
ave = self.zscore.transform(ave)
|
|
||||||
phat = self.regressor.predict(ave).item()
|
|
||||||
phat = np.clip((phat-self.correction_0)/(self.correction_1-self.correction_0), 0, 1)
|
|
||||||
return np.asarray([1-phat, phat])
|
|
||||||
|
|
||||||
def set_params(self, **parameters):
|
|
||||||
self.learner.set_params(**parameters)
|
|
||||||
|
|
||||||
def get_params(self, deep=True):
|
|
||||||
return self.learner.get_params(deep=deep)
|
|
||||||
|
|
||||||
|
|
||||||
class WinnowOrthogonal(BaseEstimator):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def fit(self, X, y):
|
|
||||||
self.classes_ = np.asarray(sorted(np.unique(y)))
|
|
||||||
w1 = np.asarray(X[y == 0].mean(axis=0)).flatten()
|
|
||||||
w2 = np.asarray(X[y == 1].mean(axis=0)).flatten()
|
|
||||||
diff = w2 - w1
|
|
||||||
orth = np.ones_like(diff)
|
|
||||||
orth[0] = -diff[1:].sum() / diff[0]
|
|
||||||
orth /= np.linalg.norm(orth)
|
|
||||||
self.w = orth
|
|
||||||
self.b = w1.dot(orth)
|
|
||||||
return self
|
|
||||||
|
|
||||||
def decision_function(self, X):
|
|
||||||
if issparse(X):
|
|
||||||
Z = X.dot(csr_matrix(self.w).T).toarray().flatten()
|
|
||||||
return Z - self.b
|
|
||||||
else:
|
|
||||||
return np.matmul(X, self.w) - self.b
|
|
||||||
|
|
||||||
def predict(self, X):
|
|
||||||
return 1 * (self.decision_function(X) > 0)
|
|
||||||
|
|
||||||
def split(self, X, y):
|
|
||||||
s = self.predict(X)
|
|
||||||
X0a = X[np.logical_and(y == 0, s == 0)]
|
|
||||||
X0b = X[np.logical_and(y == 0, s == 1)]
|
|
||||||
X1a = X[np.logical_and(y == 1, s == 0)]
|
|
||||||
X1b = X[np.logical_and(y == 1, s == 1)]
|
|
||||||
y0a = np.zeros(X0a.shape[0], dtype=np.int)
|
|
||||||
y0b = np.zeros(X0b.shape[0], dtype=np.int)
|
|
||||||
y1a = np.ones(X1a.shape[0], dtype=np.int)
|
|
||||||
y1b = np.ones(X1b.shape[0], dtype=np.int)
|
|
||||||
return X0a, X0b, X1a, X1b, y0a, y0b, y1a, y1b
|
|
||||||
|
|
||||||
def get_params(self):
|
|
||||||
return {}
|
|
||||||
|
|
||||||
def set_params(self, **params):
|
|
||||||
pass
|
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
from sklearn.linear_model import LogisticRegression
|
|
||||||
import quapy as qp
|
|
||||||
from classification.methods import PCALR
|
|
||||||
from method.meta import QuaNet
|
|
||||||
from quapy.method.aggregative import *
|
|
||||||
from NewMethods.methods import *
|
|
||||||
from experiments import run, SAMPLE_SIZE
|
|
||||||
import numpy as np
|
|
||||||
import itertools
|
|
||||||
from joblib import Parallel, delayed
|
|
||||||
import settings
|
|
||||||
import argparse
|
|
||||||
import torch
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Run experiments for Tweeter Sentiment Quantification')
|
|
||||||
parser.add_argument('results', metavar='RESULT_PATH', type=str, help='path to the directory where to store the results')
|
|
||||||
#parser.add_argument('svmperfpath', metavar='SVMPERF_PATH', type=str, help='path to the directory with svmperf')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def quantification_models():
|
|
||||||
def newLR():
|
|
||||||
return LogisticRegression(max_iter=1000, solver='lbfgs', n_jobs=-1)
|
|
||||||
__C_range = np.logspace(-4, 5, 10)
|
|
||||||
lr_params = {'C': __C_range, 'class_weight': [None, 'balanced']}
|
|
||||||
svmperf_params = {'C': __C_range}
|
|
||||||
#yield 'paccsld', PACCSLD(newLR()), lr_params
|
|
||||||
yield 'hdysld', OneVsAll(HDySLD(newLR())), lr_params # <-- promising!
|
|
||||||
|
|
||||||
#device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
|
||||||
#print(f'Running QuaNet in {device}')
|
|
||||||
#yield 'quanet', QuaNet(PCALR(**newLR().get_params()), SAMPLE_SIZE, device=device), lr_params
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
print(f'Result folder: {args.results}')
|
|
||||||
np.random.seed(0)
|
|
||||||
|
|
||||||
optim_losses = ['mae']
|
|
||||||
datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TRAIN
|
|
||||||
models = quantification_models()
|
|
||||||
|
|
||||||
results = Parallel(n_jobs=settings.N_JOBS)(
|
|
||||||
delayed(run)(experiment) for experiment in itertools.product(optim_losses, datasets, models)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,148 +0,0 @@
|
||||||
import quapy as qp
|
|
||||||
import numpy as np
|
|
||||||
from os import makedirs
|
|
||||||
import sys, os
|
|
||||||
import pickle
|
|
||||||
from experiments import result_path
|
|
||||||
from gen_tables import save_table, experiment_errors
|
|
||||||
from tabular import Table
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
tables_path = './tables'
|
|
||||||
MAXTONE = 50 # sets the intensity of the maximum color reached by the worst (red) and best (green) results
|
|
||||||
|
|
||||||
makedirs(tables_path, exist_ok=True)
|
|
||||||
|
|
||||||
sample_size = 100
|
|
||||||
qp.environ['SAMPLE_SIZE'] = sample_size
|
|
||||||
|
|
||||||
|
|
||||||
nice = {
|
|
||||||
'mae':'AE',
|
|
||||||
'mrae':'RAE',
|
|
||||||
'ae':'AE',
|
|
||||||
'rae':'RAE',
|
|
||||||
'svmkld': 'SVM(KLD)',
|
|
||||||
'svmnkld': 'SVM(NKLD)',
|
|
||||||
'svmq': 'SVM(Q)',
|
|
||||||
'svmae': 'SVM(AE)',
|
|
||||||
'svmnae': 'SVM(NAE)',
|
|
||||||
'svmmae': 'SVM(AE)',
|
|
||||||
'svmmrae': 'SVM(RAE)',
|
|
||||||
'quanet': 'QuaNet',
|
|
||||||
'hdy': 'HDy',
|
|
||||||
'hdysld': 'HDy-SLD',
|
|
||||||
'dys': 'DyS',
|
|
||||||
'svmperf':'',
|
|
||||||
'sanders': 'Sanders',
|
|
||||||
'semeval13': 'SemEval13',
|
|
||||||
'semeval14': 'SemEval14',
|
|
||||||
'semeval15': 'SemEval15',
|
|
||||||
'semeval16': 'SemEval16',
|
|
||||||
'Average': 'Average'
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def nicerm(key):
|
|
||||||
return '\mathrm{'+nice[key]+'}'
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = argparse.ArgumentParser(description='Generate tables for Tweeter Sentiment Quantification')
|
|
||||||
parser.add_argument('results', metavar='RESULT_PATH', type=str,
|
|
||||||
help='path to the directory containing the results of the methods tested in Gao & Sebastiani')
|
|
||||||
parser.add_argument('newresults', metavar='RESULT_PATH', type=str,
|
|
||||||
help='path to the directory containing the results for the experimental methods')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST
|
|
||||||
evaluation_measures = [qp.error.ae, qp.error.rae]
|
|
||||||
gao_seb_methods = ['cc', 'acc', 'pcc', 'pacc', 'sld', 'svmq', 'svmkld', 'svmnkld']
|
|
||||||
new_methods = ['hdy'] # methods added to the Gao & Sebastiani methods
|
|
||||||
experimental_methods = ['hdysld'] # experimental
|
|
||||||
|
|
||||||
for i, eval_func in enumerate(evaluation_measures):
|
|
||||||
|
|
||||||
# Tables evaluation scores for AE and RAE (two tables)
|
|
||||||
# ----------------------------------------------------
|
|
||||||
|
|
||||||
eval_name = eval_func.__name__
|
|
||||||
|
|
||||||
added_methods = ['svmm' + eval_name] + new_methods
|
|
||||||
methods = gao_seb_methods + added_methods + experimental_methods
|
|
||||||
nold_methods = len(gao_seb_methods)
|
|
||||||
nnew_methods = len(added_methods)
|
|
||||||
nexp_methods = len(experimental_methods)
|
|
||||||
|
|
||||||
# fill data table
|
|
||||||
table = Table(benchmarks=datasets, methods=methods)
|
|
||||||
for dataset in datasets:
|
|
||||||
for method in methods:
|
|
||||||
if method in experimental_methods:
|
|
||||||
path = args.newresults
|
|
||||||
else:
|
|
||||||
path = args.results
|
|
||||||
table.add(dataset, method, experiment_errors(path, dataset, method, eval_name))
|
|
||||||
|
|
||||||
# write the latex table
|
|
||||||
tabular = """
|
|
||||||
\\begin{tabularx}{\\textwidth}{|c||""" + ('Y|'*nold_methods) + '|' + ('Y|'*nnew_methods) + '|' + ('Y|'*nexp_methods) + """} \hline
|
|
||||||
& \multicolumn{"""+str(nold_methods)+"""}{c||}{Methods tested in~\cite{Gao:2016uq}} &
|
|
||||||
\multicolumn{"""+str(nnew_methods)+"""}{c|}{} &
|
|
||||||
\multicolumn{"""+str(nexp_methods)+"""}{c|}{}\\\\ \hline
|
|
||||||
"""
|
|
||||||
rowreplace={dataset: nice.get(dataset, dataset.upper()) for dataset in datasets}
|
|
||||||
colreplace={method:'\side{' + nice.get(method, method.upper()) +'$^{' + nicerm(eval_name) + '}$} ' for method in methods}
|
|
||||||
|
|
||||||
tabular += table.latexTabular(benchmark_replace=rowreplace, method_replace=colreplace)
|
|
||||||
tabular += "\n\end{tabularx}"
|
|
||||||
|
|
||||||
save_table(f'./tables/tab_results_{eval_name}.new.tex', tabular)
|
|
||||||
|
|
||||||
# Tables ranks for AE and RAE (two tables)
|
|
||||||
# ----------------------------------------------------
|
|
||||||
# fill the data table
|
|
||||||
ranktable = Table(benchmarks=datasets, methods=methods, missing='--')
|
|
||||||
for dataset in datasets:
|
|
||||||
for method in methods:
|
|
||||||
ranktable.add(dataset, method, values=table.get(dataset, method, 'rank'))
|
|
||||||
|
|
||||||
# write the latex table
|
|
||||||
tabular = """
|
|
||||||
\\begin{tabularx}{\\textwidth}{|c||""" + ('Y|'*nold_methods) + '|' + ('Y|'*nnew_methods) + '|' + ('Y|'*nexp_methods) + """} \hline
|
|
||||||
& \multicolumn{"""+str(nold_methods)+"""}{c||}{Methods tested in~\cite{Gao:2016uq}} &
|
|
||||||
\multicolumn{"""+str(nnew_methods)+"""}{c|}{} &
|
|
||||||
\multicolumn{"""+str(nexp_methods)+"""}{c|}{}\\\\ \hline
|
|
||||||
"""
|
|
||||||
for method in methods:
|
|
||||||
tabular += ' & \side{' + nice.get(method, method.upper()) +'$^{' + nicerm(eval_name) + '}$} '
|
|
||||||
tabular += '\\\\\hline\n'
|
|
||||||
|
|
||||||
for dataset in datasets:
|
|
||||||
tabular += nice.get(dataset, dataset.upper()) + ' '
|
|
||||||
for method in methods:
|
|
||||||
newrank = ranktable.get(dataset, method)
|
|
||||||
if newrank != '--':
|
|
||||||
newrank = f'{int(newrank)}'
|
|
||||||
color = ranktable.get_color(dataset, method)
|
|
||||||
if color == '--':
|
|
||||||
color = ''
|
|
||||||
tabular += ' & ' + f'{newrank}' + color
|
|
||||||
tabular += '\\\\\hline\n'
|
|
||||||
tabular += '\hline\n'
|
|
||||||
|
|
||||||
tabular += 'Average '
|
|
||||||
for method in methods:
|
|
||||||
newrank = ranktable.get_average(method)
|
|
||||||
if newrank != '--':
|
|
||||||
newrank = f'{newrank:.1f}'
|
|
||||||
color = ranktable.get_average(method, 'color')
|
|
||||||
if color == '--':
|
|
||||||
color = ''
|
|
||||||
tabular += ' & ' + f'{newrank}' + color
|
|
||||||
tabular += '\\\\\hline\n'
|
|
||||||
tabular += "\end{tabularx}"
|
|
||||||
|
|
||||||
save_table(f'./tables/tab_rank_{eval_name}.new.tex', tabular)
|
|
||||||
|
|
||||||
print("[Done]")
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
import multiprocessing
|
|
||||||
|
|
||||||
N_JOBS = -2 #multiprocessing.cpu_count()
|
|
||||||
SAMPLE_SIZE = 100
|
|
||||||
Loading…
Reference in New Issue