Compare commits
No commits in common. "b823745dd132db5223fd3d6815cf3bd49b6ca8b4" and "01ef81bf254f1d0a5f70f8727929d82ac8e48f43" have entirely different histories.
b823745dd1
...
01ef81bf25
|
|
@ -6,14 +6,9 @@ from distribution_matching.method_kdey_closed_efficient_correct import KDEyclose
|
|||
from quapy.method.aggregative import EMQ, CC, PCC, DistributionMatching, PACC, HDy, OneVsAllAggregative, ACC
|
||||
from distribution_matching.method_dirichlety import DIRy
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from distribution_matching.method_kdey_closed_efficient import KDEyclosed_efficient
|
||||
|
||||
# the full list of methods tested in the paper (reported in the appendix)
|
||||
METHODS = ['ACC', 'PACC', 'HDy-OvA', 'DM-T', 'DM-HD', 'KDEy-HD', 'DM-CS', 'KDEy-CS', 'DIR', 'EMQ', 'EMQ-BCTS', 'KDEy-ML']
|
||||
|
||||
# uncomment this other list for the methods shown in the body of the paper (the other methods are not comparable in performance)
|
||||
#METHODS = ['PACC', 'DM-T', 'DM-HD', 'KDEy-HD', 'DM-CS', 'KDEy-CS', 'EMQ', 'KDEy-ML']
|
||||
from method_kdey_closed_efficient import KDEyclosed_efficient
|
||||
|
||||
METHODS = ['ACC', 'PACC', 'HDy-OvA', 'DM-T', 'DM-HD', 'KDEy-DMhd4', 'DM-CS', 'KDEy-closed++', 'DIR', 'EMQ', 'KDEy-ML'] #['ACC', 'PACC', 'HDy-OvA', 'DIR', 'DM', 'KDEy-DMhd3', 'KDEy-closed++', 'EMQ', 'KDEy-ML'] #, 'KDEy-DMhd2'] #, 'KDEy-DMhd2', 'DM-HD'] 'KDEy-DMjs', 'KDEy-DM', 'KDEy-ML+', 'KDEy-DMhd3+', 'EMQ-C',
|
||||
BIN_METHODS = [x.replace('-OvA', '') for x in METHODS]
|
||||
|
||||
|
||||
|
|
@ -22,12 +17,6 @@ hyper_LR = {
|
|||
'classifier__class_weight': ['balanced', None]
|
||||
}
|
||||
|
||||
hyper_kde = {
|
||||
'bandwidth': np.linspace(0.01, 0.2, 20)
|
||||
}
|
||||
|
||||
nbins_range = [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 64]
|
||||
|
||||
def new_method(method, **lr_kwargs):
|
||||
|
||||
lr = LogisticRegression(**lr_kwargs)
|
||||
|
|
@ -44,22 +33,33 @@ def new_method(method, **lr_kwargs):
|
|||
elif method == 'PACC':
|
||||
param_grid = hyper_LR
|
||||
quantifier = PACC(lr)
|
||||
elif method in ['KDEy-HD']:
|
||||
param_grid = {**hyper_kde, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=10000, val_split=10)
|
||||
elif method == 'KDEy-CS':
|
||||
param_grid = {**hyper_kde, **hyper_LR}
|
||||
quantifier = KDEyclosed_efficient_corr(lr, val_split=10)
|
||||
elif method == 'KDEy-ML':
|
||||
param_grid = {**hyper_kde, **hyper_LR}
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='max_likelihood', val_split=10)
|
||||
elif method == 'KDEy-closed':
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEyclosed(lr, val_split=10)
|
||||
elif method == 'KDEy-closed+':
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEyclosed_efficient(lr, val_split=10)
|
||||
elif method == 'KDEy-closed++':
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEyclosed_efficient_corr(lr, val_split=10)
|
||||
elif method in ['KDEy-DM']:
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence', divergence='l2', montecarlo_trials=5000, val_split=10)
|
||||
elif method == 'DIR':
|
||||
param_grid = hyper_LR
|
||||
quantifier = DIRy(lr)
|
||||
elif method == 'EMQ':
|
||||
param_grid = hyper_LR
|
||||
quantifier = EMQ(lr)
|
||||
elif method == 'EMQ-BCTS':
|
||||
elif method == 'EMQ-C':
|
||||
method_params = {'exact_train_prev': [False], 'recalib': ['bcts']}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = EMQ(lr)
|
||||
|
|
@ -69,9 +69,17 @@ def new_method(method, **lr_kwargs):
|
|||
elif method == 'HDy-OvA':
|
||||
param_grid = {'binary_quantifier__' + key: val for key, val in hyper_LR.items()}
|
||||
quantifier = OneVsAllAggregative(HDy(lr))
|
||||
elif method == 'DM':
|
||||
method_params = {
|
||||
'nbins': [4,8,16,32],
|
||||
'val_split': [10, 0.4],
|
||||
'divergence': ['HD', 'topsoe', 'l2']
|
||||
}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = DistributionMatching(lr)
|
||||
elif method == 'DM-T':
|
||||
method_params = {
|
||||
'nbins': nbins_range,
|
||||
'nbins': [2,3,4,5,6,7,8,9,10,12,14,16,18,20,22,24,26,28,30,32,64],
|
||||
'val_split': [10],
|
||||
'divergence': ['topsoe']
|
||||
}
|
||||
|
|
@ -79,7 +87,7 @@ def new_method(method, **lr_kwargs):
|
|||
quantifier = DistributionMatching(lr)
|
||||
elif method == 'DM-HD':
|
||||
method_params = {
|
||||
'nbins': nbins_range,
|
||||
'nbins': [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 64],
|
||||
'val_split': [10],
|
||||
'divergence': ['HD']
|
||||
}
|
||||
|
|
@ -87,12 +95,55 @@ def new_method(method, **lr_kwargs):
|
|||
quantifier = DistributionMatching(lr)
|
||||
elif method == 'DM-CS':
|
||||
method_params = {
|
||||
'nbins': nbins_range,
|
||||
'nbins': [2,3,4,5,6,7,8,9,10,12,14,16,18,20,22,24,26,28,30,32,64],
|
||||
'val_split': [10],
|
||||
'divergence': ['CS']
|
||||
}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = DistributionMatching(lr)
|
||||
|
||||
# experimental
|
||||
elif method in ['KDEy-DMkld']:
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence', divergence='KLD', montecarlo_trials=5000, val_split=10)
|
||||
# elif method in ['KDEy-DMhd']:
|
||||
# The code to reproduce this run is commented in the min_divergence target, I think it was incorrect...
|
||||
# method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
# param_grid = {**method_params, **hyper_LR}
|
||||
# quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
|
||||
elif method in ['KDEy-DMhd2']:
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence_uniform', divergence='HD', montecarlo_trials=5000, val_split=10)
|
||||
elif method in ['KDEy-DMjs']:
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence_uniform', divergence='JS', montecarlo_trials=5000, val_split=10)
|
||||
elif method in ['KDEy-DMhd3']:
|
||||
# I have realized that there was an error. I am sampling from the validation distribution (V) and not from the
|
||||
# test distribution (T) just because the validation can be sampled in fit only once and pre-computed densities
|
||||
# can be stored. This means that the reference distribution is V and not T. Then I have found that an
|
||||
# f-divergence is defined as D(p||q) \int_{R^n}q(x)f(p(x)/q(x))dx = E_{x~q}[f(p(x)/q(x))], so if I am sampling
|
||||
# V then I am computing D(T||V) (and not D(V||T) as I thought).
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
|
||||
elif method in ['KDEy-DMhd4']:
|
||||
# This is the new version in which we apply importance sampling, i.e., we compute:
|
||||
# D(p_a||q) = 1/N sum_x f(p(x)/q(x)) * (q(x)/r(x))
|
||||
# where x ~iid r, with r = p_u, and u = (1/n, 1/n, ..., 1/n) the uniform vector
|
||||
method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
|
||||
elif method == 'DM-HD':
|
||||
method_params = {
|
||||
'nbins': [4,8,16,32],
|
||||
'val_split': [10, 0.4],
|
||||
}
|
||||
param_grid = {**method_params, **hyper_LR}
|
||||
quantifier = DistributionMatching(lr, divergence='HD')
|
||||
|
||||
else:
|
||||
raise NotImplementedError('unknown method', method)
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ for i, post_set in enumerate([post_c1, post_c2, post_c3, post_test]):
|
|||
ax = fig.add_subplot(141+i, projection='3d')
|
||||
for post, c, z in zip(post_set.T, colors, positions):
|
||||
|
||||
hist, bins = np.histogram(post, bins=nbins, density=True, range=[0,1])
|
||||
hist, bins = np.histogram(post, bins=nbins, density=True)
|
||||
xs = (bins[:-1] + bins[1:])/2
|
||||
|
||||
ax.bar(xs, hist, width=1/nbins, zs=z, zdir='y', color=c, ec=c, alpha=0.6)
|
||||
|
|
|
|||
|
|
@ -129,8 +129,7 @@ class KDEy(AggregativeProbabilisticQuantifier):
|
|||
elif self.target == 'min_divergence':
|
||||
N = self.montecarlo_trials
|
||||
rs = self.random_state
|
||||
n = data.n_classes
|
||||
self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.val_densities])
|
||||
self.reference_samples = np.vstack([kde_i.sample(N, random_state=rs) for kde_i in self.val_densities])
|
||||
self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.val_densities])
|
||||
self.reference_density = np.mean(self.reference_classwise_densities, axis=0) # equiv. to (uniform @ self.reference_classwise_densities)
|
||||
elif self.target == 'min_divergence_deprecated': # the version of the first draft, with n*N presampled, then alpha*N chosen for class
|
||||
|
|
@ -171,48 +170,15 @@ class KDEy(AggregativeProbabilisticQuantifier):
|
|||
else:
|
||||
raise ValueError('only squared HD is currently implemented')
|
||||
|
||||
epsilon = 1e-10
|
||||
qs = test_densities + epsilon
|
||||
rs = self.reference_density + epsilon
|
||||
iw = qs/rs #importance weights
|
||||
p_class = self.reference_classwise_densities + epsilon
|
||||
fracs = p_class/qs
|
||||
|
||||
def divergence(prev):
|
||||
# ps / qs = (prev @ p_class) / qs = prev @ (p_class / qs) = prev @ fracs
|
||||
ps_div_qs = prev @ fracs
|
||||
return np.mean( f(ps_div_qs) * iw )
|
||||
|
||||
return F.optim_minimize(divergence, n_classes)
|
||||
|
||||
# new version in which we retain all n*N examples (sampled from a mixture with uniform parameter), and then
|
||||
# apply importance sampling (IS). In this version we compute D(q||p_alpha) with IS, and not D(p_alpha||q) as
|
||||
# in the reformulation proposed above
|
||||
def _target_divergence_q__p(self, posteriors):
|
||||
# in this variant we evaluate the divergence using a Montecarlo approach
|
||||
n_classes = len(self.val_densities)
|
||||
|
||||
test_kde = self.get_kde_function(posteriors)
|
||||
test_densities = self.pdf(test_kde, self.reference_samples)
|
||||
|
||||
def f_squared_hellinger(u):
|
||||
return (np.sqrt(u)-1)**2
|
||||
|
||||
# todo: this will fail when self.divergence is a callable, and is not the right place to do it anyway
|
||||
if self.divergence.lower() == 'hd':
|
||||
f = f_squared_hellinger
|
||||
else:
|
||||
raise ValueError('only squared HD is currently implemented')
|
||||
|
||||
epsilon = 1e-10
|
||||
qs = test_densities + epsilon
|
||||
rs = self.reference_density + epsilon
|
||||
p_class = self.reference_classwise_densities + epsilon
|
||||
|
||||
# D(q||p_a) = 1/N sum f(q/p_a) * (p_a / p_u)
|
||||
|
||||
def divergence(prev):
|
||||
p_a = prev @ p_class
|
||||
return np.mean( f(qs/p_a) * (p_a/rs) )
|
||||
ps = prev @ self.reference_classwise_densities + epsilon
|
||||
return np.mean( f(ps/qs) * (qs/rs) )
|
||||
|
||||
return F.optim_minimize(divergence, n_classes)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,249 +0,0 @@
|
|||
from distribution_matching.commons import BIN_METHODS, METHODS
|
||||
import quapy as qp
|
||||
from os import makedirs
|
||||
import os
|
||||
|
||||
from tabular import Table
|
||||
import pandas as pd
|
||||
|
||||
tables_path = '.'
|
||||
# makedirs(tables_path, exist_ok=True)
|
||||
|
||||
MAXTONE = 35 # sets the intensity of the maximum color reached by the worst (red) and best (green) results
|
||||
SHOW_STD = False
|
||||
|
||||
NUM_ADJUSTMENT_METHODS = 2 if 'ACC' in METHODS else 1
|
||||
NUM_MAXIMUM_LIKELIHOOD_METHODS = 4 if 'DIR' in METHODS else 3
|
||||
NUM_DISTRIBUTION_MATCHING_PAIRS = 2
|
||||
NUM_DISTRIBUTION_MATCHING_METHODS = NUM_DISTRIBUTION_MATCHING_PAIRS*2 + (2 if 'HDy-OvA' in METHODS else 1)
|
||||
|
||||
qp.environ['SAMPLE_SIZE'] = 100
|
||||
|
||||
nice_bench = {
|
||||
'sanders': 'Sanders',
|
||||
'semeval13': 'SemEval13',
|
||||
'semeval14': 'SemEval14',
|
||||
'semeval15': 'SemEval15',
|
||||
'semeval16': 'SemEval16',
|
||||
}
|
||||
|
||||
nice_method={
|
||||
'KDEy-MLE': 'KDEy-ML',
|
||||
'KDEy-DMhd4': 'KDEy-HD',
|
||||
'KDEy-closed++': 'KDEy-CS',
|
||||
'EMQ-C': 'EMQ-BCTS'
|
||||
}
|
||||
|
||||
def save_table(path, table):
|
||||
print(f'saving results in {path}')
|
||||
with open(path, 'wt') as foo:
|
||||
foo.write(table)
|
||||
|
||||
|
||||
def nicerm(key):
|
||||
return '\mathrm{'+nice[key]+'}'
|
||||
|
||||
|
||||
def make_table(tabs, eval, benchmark_groups, benchmark_names, compact=False):
|
||||
|
||||
n_methods = len(METHODS)
|
||||
assert n_methods == (NUM_ADJUSTMENT_METHODS+NUM_DISTRIBUTION_MATCHING_METHODS+NUM_MAXIMUM_LIKELIHOOD_METHODS), \
|
||||
"Unexpected number of methods"
|
||||
|
||||
cline = "\cline{2-" + str(n_methods+ 1) + "}"
|
||||
|
||||
# write the latex table
|
||||
tabular = """
|
||||
\\begin{tabular}{|c|""" + ('c|' * NUM_ADJUSTMENT_METHODS) + 'c|c' + ('|c|c' * (NUM_DISTRIBUTION_MATCHING_PAIRS)) + ('|c' * NUM_MAXIMUM_LIKELIHOOD_METHODS) + """|} """ + cline + """
|
||||
\multicolumn{1}{c}{} &
|
||||
\multicolumn{"""+str(NUM_ADJUSTMENT_METHODS)+"""}{|c}{Adjustment} &
|
||||
\multicolumn{"""+str(NUM_DISTRIBUTION_MATCHING_METHODS)+"""}{|c|}{Distribution Matching} &
|
||||
\multicolumn{"""+str(NUM_MAXIMUM_LIKELIHOOD_METHODS)+"""}{c|}{Maximum Likelihood} \\\\
|
||||
\hline
|
||||
"""
|
||||
for i, (tab, group, name) in enumerate(zip(tabs, benchmark_groups, benchmark_names)):
|
||||
tablines = tab.latexTabular(benchmark_replace=nice_bench, method_replace=nice_method, endl='\\\\'+ cline, aslines=True)
|
||||
print(tablines)
|
||||
tablines[0] = tablines[0].replace('\multicolumn{1}{c|}{}', '\\textbf{'+name+'}')
|
||||
if not compact:
|
||||
tabular += '\n'.join(tablines)
|
||||
else:
|
||||
# if compact, keep the method names and the average; discard the rest
|
||||
tabular += tablines[0] + '\n' + tablines[-1] + '\n'
|
||||
|
||||
tabular += "\n" + "\\textit{Rank} & " + tab.getRankTable(prec_mean=0 if name.startswith('LeQua') else 1).latexAverage()
|
||||
if i < (len(tabs) - 1):
|
||||
tabular += "\\hline\n"
|
||||
else:
|
||||
tabular += "\n"
|
||||
tabular += "\end{tabular}"
|
||||
return tabular
|
||||
|
||||
|
||||
def gen_tables_uci_multiclass(eval):
|
||||
|
||||
print('Generating table for UCI Multiclass Datasets', eval)
|
||||
dir_results = f'../results/ucimulti/{eval}'
|
||||
|
||||
datasets = qp.datasets.UCI_MULTICLASS_DATASETS
|
||||
|
||||
tab = Table(
|
||||
benchmarks=datasets,
|
||||
methods=METHODS,
|
||||
ttest='wilcoxon',
|
||||
prec_mean=4,
|
||||
show_std=SHOW_STD,
|
||||
prec_std=4,
|
||||
clean_zero=(eval=='mae'),
|
||||
average=True,
|
||||
maxtone=MAXTONE
|
||||
)
|
||||
|
||||
for dataset in datasets:
|
||||
print(f'\t Dataset: {dataset}: ', end='')
|
||||
for method in METHODS:
|
||||
result_path = f'{dir_results}/{method}_{dataset}.dataframe'
|
||||
if os.path.exists(result_path):
|
||||
df = pd.read_csv(result_path)
|
||||
print(f'{method}', end=' ')
|
||||
tab.add(dataset, method, df[eval].values)
|
||||
else:
|
||||
print(f'MISSING-{method}', end=' ')
|
||||
print()
|
||||
|
||||
return tab
|
||||
|
||||
|
||||
def gen_tables_uci_bin(eval):
|
||||
|
||||
print('Generating table for UCI Datasets', eval)
|
||||
dir_results = f'../results/binary/{eval}'
|
||||
|
||||
exclude = ['acute.a', 'acute.b', 'iris.1', 'balance.2']
|
||||
datasets = [x for x in qp.datasets.UCI_DATASETS if x not in exclude]
|
||||
|
||||
tab = Table(
|
||||
benchmarks=datasets,
|
||||
methods=BIN_METHODS,
|
||||
ttest='wilcoxon',
|
||||
prec_mean=4,
|
||||
show_std=SHOW_STD,
|
||||
prec_std=4,
|
||||
clean_zero=(eval=='mae'),
|
||||
average=True,
|
||||
maxtone=MAXTONE
|
||||
)
|
||||
|
||||
for dataset in datasets:
|
||||
print(f'\t Dataset: {dataset}: ', end='')
|
||||
for method in BIN_METHODS:
|
||||
result_path = f'{dir_results}/{method}_{dataset}.dataframe'
|
||||
if os.path.exists(result_path):
|
||||
df = pd.read_csv(result_path)
|
||||
print(f'{method}', end=' ')
|
||||
tab.add(dataset, method, df[eval].values)
|
||||
else:
|
||||
print(f'MISSING-{method}', end=' ')
|
||||
|
||||
return tab
|
||||
|
||||
|
||||
|
||||
def gen_tables_tweet(eval):
|
||||
|
||||
print('Generating table for Twitter', eval)
|
||||
dir_results = f'../results/tweet/{eval}'
|
||||
|
||||
datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST
|
||||
|
||||
tab = Table(
|
||||
benchmarks=datasets,
|
||||
methods=METHODS,
|
||||
ttest='wilcoxon',
|
||||
prec_mean=4,
|
||||
show_std=SHOW_STD,
|
||||
prec_std=4,
|
||||
clean_zero=(eval=='mae'),
|
||||
average=True,
|
||||
maxtone=MAXTONE
|
||||
)
|
||||
|
||||
for dataset in datasets:
|
||||
print(f'\t Dataset: {dataset}: ', end='')
|
||||
for method in METHODS:
|
||||
result_path = f'{dir_results}/{method}_{dataset}.dataframe'
|
||||
if os.path.exists(result_path):
|
||||
df = pd.read_csv(result_path)
|
||||
print(f'{method}', end=' ')
|
||||
tab.add(dataset, method, df[eval].values)
|
||||
else:
|
||||
print(f'MISSING-{method}', end=' ')
|
||||
print()
|
||||
|
||||
return tab
|
||||
|
||||
|
||||
def gen_tables_lequa(Methods, task, eval):
|
||||
# generating table for LeQua-T1A or Lequa-T1B; only one table with two rows, one for MAE, another for MRAE
|
||||
dataset_name = 'LeQua-'+task
|
||||
|
||||
tab = Table(
|
||||
benchmarks=[f'Average'],
|
||||
methods=Methods,
|
||||
ttest='wilcoxon',
|
||||
prec_mean=5,
|
||||
show_std=SHOW_STD,
|
||||
prec_std=4,
|
||||
clean_zero=False,
|
||||
average=False,
|
||||
maxtone=MAXTONE
|
||||
)
|
||||
|
||||
print('Generating table for T1A@Lequa', eval, end='')
|
||||
dir_results = f'../results/lequa/{task}/{eval}'
|
||||
|
||||
for method in Methods:
|
||||
result_path = f'{dir_results}/{method}.dataframe'
|
||||
if os.path.exists(result_path):
|
||||
df = pd.read_csv(result_path)
|
||||
print(f'{method}', end=' ')
|
||||
tab.add('Average', method, df[eval].values)
|
||||
else:
|
||||
print(f'MISSING-{method}', end=' ')
|
||||
print()
|
||||
|
||||
return tab
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.makedirs('./latex', exist_ok=True)
|
||||
|
||||
for eval in ['mae', 'mrae']:
|
||||
tabs = []
|
||||
tabs.append(gen_tables_tweet(eval))
|
||||
tabs.append(gen_tables_uci_multiclass(eval))
|
||||
tabs.append(gen_tables_lequa(METHODS, 'T1B', eval))
|
||||
|
||||
names = ['Tweets', 'UCI-multi', 'LeQua-T1B']
|
||||
table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=names)
|
||||
save_table(f'./latex/multiclass_{eval}.tex', table)
|
||||
|
||||
for eval in ['mae', 'mrae']:
|
||||
tabs = []
|
||||
tabs.append(gen_tables_uci_bin(eval))
|
||||
|
||||
# print uci-binary with all datasets for the appendix
|
||||
table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=['UCI-binary'])
|
||||
save_table(f'./latex/ucibinary_{eval}.tex', table)
|
||||
|
||||
# print uci-bin compacted plus lequa-T1A for the main body
|
||||
tabs.append(gen_tables_lequa(BIN_METHODS, 'T1A', eval))
|
||||
table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=['UCI-binary', 'LeQua-T1A'], compact=True)
|
||||
save_table(f'./latex/binary_{eval}.tex', table)
|
||||
|
||||
print("[Tables Done] runing latex")
|
||||
os.chdir('./latex/')
|
||||
os.system('pdflatex tables_compact.tex')
|
||||
os.system('rm tables_compact.aux tables_compact.bbl tables_compact.blg tables_compact.log tables_compact.out tables_compact.dvi')
|
||||
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
\documentclass{article}
|
||||
|
||||
|
||||
|
||||
\usepackage[utf8]{inputenc} % allow utf-8 input
|
||||
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
|
||||
\usepackage{hyperref} % hyperlinks
|
||||
\usepackage{url} % simple URL typesetting
|
||||
\usepackage{booktabs} % professional-quality tables
|
||||
\usepackage{amsfonts} % blackboard math symbols
|
||||
\usepackage{nicefrac} % compact symbols for 1/2, etc.
|
||||
\usepackage{microtype} % microtypography
|
||||
\usepackage{lipsum}
|
||||
\usepackage{fancyhdr} % header
|
||||
\usepackage{graphicx} % graphics
|
||||
\graphicspath{{media/}} % organize your images and other figures under media/ folder
|
||||
\usepackage{amsmath}
|
||||
\usepackage{bm}
|
||||
\usepackage{tabularx}
|
||||
\usepackage{color}
|
||||
\usepackage{colortbl}
|
||||
\usepackage{xcolor}
|
||||
\usepackage{lmodern}
|
||||
|
||||
\DeclareMathOperator*{\argmax}{arg\,max}
|
||||
\DeclareMathOperator*{\argmin}{arg\,min}
|
||||
|
||||
\newif\ifdraft
|
||||
\drafttrue
|
||||
|
||||
\newcommand{\juanjo}[1]{\ifdraft{\leavevmode\color{purple}{[JJ]:
|
||||
{#1}}}\else{\vspace{0ex}}\fi}
|
||||
|
||||
\newcommand{\alex}[1]{\ifdraft{\leavevmode\color{violet}{[AM]:
|
||||
{#1}}}\else{\vspace{0ex}}\fi}
|
||||
|
||||
\newcommand{\pablo}[1]{\ifdraft{\leavevmode\color{red}{[PG]:
|
||||
{#1}}}\else{\vspace{0ex}}\fi}
|
||||
|
||||
|
||||
\title{Tables}
|
||||
|
||||
|
||||
\author{
|
||||
Alejandro Moreo
|
||||
}
|
||||
|
||||
|
||||
\begin{document}
|
||||
|
||||
\maketitle
|
||||
|
||||
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Multiclass AE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{multiclass_mae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Multiclass RAE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{multiclass_mae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Binary MAE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{binary_mae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Binary MRAE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{binary_mrae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{UCI binary full AE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{ucibinary_mae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{UCI binary full RAE}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\input{ucibinary_mrae}
|
||||
}%
|
||||
\end{table}
|
||||
|
||||
|
||||
|
||||
\end{document}
|
||||
|
||||
|
|
@ -1,348 +0,0 @@
|
|||
import numpy as np
|
||||
import itertools
|
||||
from scipy.stats import ttest_ind_from_stats, wilcoxon
|
||||
|
||||
|
||||
class Table:
|
||||
VALID_TESTS = [None, "wilcoxon", "ttest"]
|
||||
|
||||
def __init__(self, benchmarks, methods, lower_is_better=True, ttest='ttest', prec_mean=3,
|
||||
clean_zero=False, show_std=False, prec_std=3, average=True, missing=None, missing_str='--', color=True, maxtone=50):
|
||||
assert ttest in self.VALID_TESTS, f'unknown test, valid are {self.VALID_TESTS}'
|
||||
|
||||
self.benchmarks = np.asarray(benchmarks)
|
||||
self.benchmark_index = {row:i for i, row in enumerate(benchmarks)}
|
||||
|
||||
self.methods = np.asarray(methods)
|
||||
self.method_index = {col:j for j, col in enumerate(methods)}
|
||||
|
||||
self.map = {}
|
||||
# keyed (#rows,#cols)-ndarrays holding computations from self.map['values']
|
||||
self._addmap('values', dtype=object)
|
||||
self.lower_is_better = lower_is_better
|
||||
self.ttest = ttest
|
||||
self.prec_mean = prec_mean
|
||||
self.clean_zero = clean_zero
|
||||
self.show_std = show_std
|
||||
self.prec_std = prec_std
|
||||
self.add_average = average
|
||||
self.missing = missing
|
||||
self.missing_str = missing_str
|
||||
self.color = color
|
||||
self.maxtone = maxtone
|
||||
|
||||
self.touch()
|
||||
|
||||
@property
|
||||
def nbenchmarks(self):
|
||||
return len(self.benchmarks)
|
||||
|
||||
@property
|
||||
def nmethods(self):
|
||||
return len(self.methods)
|
||||
|
||||
def touch(self):
|
||||
self._modif = True
|
||||
|
||||
def update(self):
|
||||
if self._modif:
|
||||
self.compute()
|
||||
|
||||
def _getfilled(self):
|
||||
return np.argwhere(self.map['fill'])
|
||||
|
||||
@property
|
||||
def values(self):
|
||||
return self.map['values']
|
||||
|
||||
def _indexes(self):
|
||||
return itertools.product(range(self.nbenchmarks), range(self.nmethods))
|
||||
|
||||
def _addmap(self, map, dtype, func=None):
|
||||
self.map[map] = np.empty((self.nbenchmarks, self.nmethods), dtype=dtype)
|
||||
if func is None:
|
||||
return
|
||||
m = self.map[map]
|
||||
f = func
|
||||
indexes = self._indexes() if map == 'fill' else self._getfilled()
|
||||
for i, j in indexes:
|
||||
m[i, j] = f(self.values[i, j])
|
||||
|
||||
def _addrank(self):
|
||||
for i in range(self.nbenchmarks):
|
||||
filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
|
||||
col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
|
||||
ranked_cols_idx = filled_cols_idx[np.argsort(col_means)]
|
||||
if not self.lower_is_better:
|
||||
ranked_cols_idx = ranked_cols_idx[::-1]
|
||||
self.map['rank'][i, ranked_cols_idx] = np.arange(1, len(filled_cols_idx)+1)
|
||||
|
||||
def _addcolor(self):
|
||||
for i in range(self.nbenchmarks):
|
||||
filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
|
||||
if filled_cols_idx.size==0:
|
||||
continue
|
||||
col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
|
||||
#col_means = [self.map['rank'][i, j] for j in filled_cols_idx]
|
||||
|
||||
minval = min(col_means)
|
||||
maxval = max(col_means)
|
||||
|
||||
for col_idx in filled_cols_idx:
|
||||
val = self.map['mean'][i,col_idx]
|
||||
norm = (maxval - minval)
|
||||
if norm > 0:
|
||||
normval = (val - minval) / norm
|
||||
else:
|
||||
normval = 0.5
|
||||
|
||||
if self.lower_is_better:
|
||||
normval = 1 - normval
|
||||
|
||||
normval = np.clip(normval, 0,1)
|
||||
|
||||
self.map['color'][i, col_idx] = color_red2green_01(normval, self.maxtone)
|
||||
|
||||
def _run_ttest(self, row, col1, col2):
|
||||
mean1 = self.map['mean'][row, col1]
|
||||
std1 = self.map['std'][row, col1]
|
||||
nobs1 = self.map['nobs'][row, col1]
|
||||
mean2 = self.map['mean'][row, col2]
|
||||
std2 = self.map['std'][row, col2]
|
||||
nobs2 = self.map['nobs'][row, col2]
|
||||
_, p_val = ttest_ind_from_stats(mean1, std1, nobs1, mean2, std2, nobs2)
|
||||
return p_val
|
||||
|
||||
def _run_wilcoxon(self, row, col1, col2):
|
||||
values1 = self.map['values'][row, col1]
|
||||
values2 = self.map['values'][row, col2]
|
||||
try:
|
||||
_, p_val = wilcoxon(values1, values2)
|
||||
except ValueError:
|
||||
p_val = 0
|
||||
return p_val
|
||||
|
||||
def _add_statistical_test(self):
|
||||
if self.ttest is None:
|
||||
return
|
||||
self.some_similar = [False]*self.nmethods
|
||||
for i in range(self.nbenchmarks):
|
||||
filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
|
||||
if len(filled_cols_idx) <= 1:
|
||||
continue
|
||||
col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
|
||||
best_pos = filled_cols_idx[np.argmin(col_means)]
|
||||
|
||||
for j in filled_cols_idx:
|
||||
if j==best_pos:
|
||||
continue
|
||||
if self.ttest == 'ttest':
|
||||
p_val = self._run_ttest(i, best_pos, j)
|
||||
else:
|
||||
p_val = self._run_wilcoxon(i, best_pos, j)
|
||||
|
||||
pval_outcome = pval_interpretation(p_val)
|
||||
self.map['ttest'][i, j] = pval_outcome
|
||||
if pval_outcome != 'Diff':
|
||||
self.some_similar[j] = True
|
||||
|
||||
def compute(self):
|
||||
self._addmap('fill', dtype=bool, func=lambda x: x is not None)
|
||||
self._addmap('mean', dtype=float, func=np.mean)
|
||||
self._addmap('std', dtype=float, func=np.std)
|
||||
self._addmap('nobs', dtype=float, func=len)
|
||||
self._addmap('rank', dtype=int, func=None)
|
||||
self._addmap('color', dtype=object, func=None)
|
||||
self._addmap('ttest', dtype=object, func=None)
|
||||
self._addmap('latex', dtype=object, func=None)
|
||||
self._addrank()
|
||||
self._addcolor()
|
||||
self._add_statistical_test()
|
||||
if self.add_average:
|
||||
self._addave()
|
||||
self._modif = False
|
||||
|
||||
def _is_column_full(self, col):
|
||||
return all(self.map['fill'][:, self.method_index[col]])
|
||||
|
||||
def _addave(self):
|
||||
ave = Table(['ave'], self.methods,
|
||||
lower_is_better=self.lower_is_better,
|
||||
ttest=self.ttest,
|
||||
average=False,
|
||||
missing=self.missing,
|
||||
missing_str=self.missing_str,
|
||||
prec_mean=self.prec_mean,
|
||||
prec_std=self.prec_std,
|
||||
clean_zero=self.clean_zero,
|
||||
show_std=self.show_std,
|
||||
color=self.color,
|
||||
maxtone=self.maxtone)
|
||||
for col in self.methods:
|
||||
values = None
|
||||
if self._is_column_full(col):
|
||||
if self.ttest == 'ttest':
|
||||
# values = np.asarray(self.map['mean'][:, self.method_index[col]])
|
||||
values = np.concatenate(self.values[:, self.method_index[col]])
|
||||
else: # wilcoxon
|
||||
# values = np.asarray(self.map['mean'][:, self.method_index[col]])
|
||||
values = np.concatenate(self.values[:, self.method_index[col]])
|
||||
ave.add('ave', col, values)
|
||||
self.average = ave
|
||||
|
||||
def add(self, benchmark, method, values):
|
||||
if values is not None:
|
||||
values = np.asarray(values)
|
||||
if values.ndim==0:
|
||||
values = values.flatten()
|
||||
rid, cid = self._coordinates(benchmark, method)
|
||||
self.map['values'][rid, cid] = values
|
||||
self.touch()
|
||||
|
||||
def get(self, benchmark, method, attr='mean'):
|
||||
self.update()
|
||||
assert attr in self.map, f'unknwon attribute {attr}'
|
||||
rid, cid = self._coordinates(benchmark, method)
|
||||
if self.map['fill'][rid, cid]:
|
||||
v = self.map[attr][rid, cid]
|
||||
if v is None or (isinstance(v,float) and np.isnan(v)):
|
||||
return self.missing
|
||||
return v
|
||||
else:
|
||||
return self.missing
|
||||
|
||||
def _coordinates(self, benchmark, method):
|
||||
assert benchmark in self.benchmark_index, f'benchmark {benchmark} out of range'
|
||||
assert method in self.method_index, f'method {method} out of range'
|
||||
rid = self.benchmark_index[benchmark]
|
||||
cid = self.method_index[method]
|
||||
return rid, cid
|
||||
|
||||
def get_average(self, method, attr='mean'):
|
||||
self.update()
|
||||
if self.add_average:
|
||||
return self.average.get('ave', method, attr=attr)
|
||||
return None
|
||||
|
||||
def get_color(self, benchmark, method):
|
||||
color = self.get(benchmark, method, attr='color')
|
||||
if color is None:
|
||||
return ''
|
||||
return color
|
||||
|
||||
def latex(self, benchmark, method):
|
||||
self.update()
|
||||
i,j = self._coordinates(benchmark, method)
|
||||
if self.map['fill'][i,j] == False:
|
||||
return self.missing_str
|
||||
|
||||
mean = self.map['mean'][i,j]
|
||||
l = f" {mean:.{self.prec_mean}f}"
|
||||
if self.clean_zero:
|
||||
l = l.replace(' 0.', '.')
|
||||
|
||||
isbest = self.map['rank'][i,j] == 1
|
||||
if isbest:
|
||||
l = "\\textbf{"+l.strip()+"}"
|
||||
|
||||
stat = '' if self.ttest is None else '^{\phantom{\ddag}}'
|
||||
if self.ttest is not None and self.some_similar[j]:
|
||||
test_label = self.map['ttest'][i,j]
|
||||
if test_label == 'Sim':
|
||||
stat = '^{\dag}'
|
||||
elif test_label == 'Same':
|
||||
stat = '^{\ddag}'
|
||||
elif isbest or test_label == 'Diff':
|
||||
stat = '^{\phantom{\ddag}}'
|
||||
|
||||
std = ''
|
||||
if self.show_std:
|
||||
std = self.map['std'][i,j]
|
||||
std = f" {std:.{self.prec_std}f}"
|
||||
if self.clean_zero:
|
||||
std = std.replace(' 0.', '.')
|
||||
std = f"\pm {std:{self.prec_std}}"
|
||||
|
||||
if stat!='' or std!='':
|
||||
l = f'{l}${stat}{std}$'
|
||||
|
||||
if self.color:
|
||||
l += ' ' + self.map['color'][i,j]
|
||||
|
||||
return l
|
||||
|
||||
def latexTabular(self, benchmark_replace={}, method_replace={}, aslines=False, endl='\\\\\hline'):
|
||||
lines = []
|
||||
l = '\multicolumn{1}{c|}{} & '
|
||||
l += ' & '.join([method_replace.get(col, col) for col in self.methods])
|
||||
l += ' \\\\\hline'
|
||||
lines.append(l)
|
||||
|
||||
for row in self.benchmarks:
|
||||
rowname = benchmark_replace.get(row, row)
|
||||
l = rowname + ' & '
|
||||
l += self.latexRow(row, endl=endl)
|
||||
lines.append(l)
|
||||
|
||||
if self.add_average:
|
||||
# l += '\hline\n'
|
||||
l = '\hline \n \\textit{Average} & '
|
||||
l += self.latexAverage(endl=endl)
|
||||
lines.append(l)
|
||||
if not aslines:
|
||||
lines='\n'.join(lines)
|
||||
return lines
|
||||
|
||||
def latexRow(self, benchmark, endl='\\\\\hline\n'):
|
||||
s = [self.latex(benchmark, col) for col in self.methods]
|
||||
s = ' & '.join(s)
|
||||
s += ' ' + endl
|
||||
return s
|
||||
|
||||
def latexAverage(self, endl='\\\\\hline\n'):
|
||||
if self.add_average:
|
||||
return self.average.latexRow('ave', endl=endl)
|
||||
|
||||
def getRankTable(self, prec_mean=0):
|
||||
t = Table(benchmarks=self.benchmarks, methods=self.methods, prec_mean=prec_mean, average=True, maxtone=self.maxtone, ttest=None)
|
||||
for rid, cid in self._getfilled():
|
||||
row = self.benchmarks[rid]
|
||||
col = self.methods[cid]
|
||||
t.add(row, col, self.get(row, col, 'rank'))
|
||||
t.compute()
|
||||
return t
|
||||
|
||||
def dropMethods(self, methods):
|
||||
drop_index = [self.method_index[m] for m in methods]
|
||||
new_methods = np.delete(self.methods, drop_index)
|
||||
new_index = {col:j for j, col in enumerate(new_methods)}
|
||||
|
||||
self.map['values'] = self.values[:,np.asarray([self.method_index[m] for m in new_methods], dtype=int)]
|
||||
self.methods = new_methods
|
||||
self.method_index = new_index
|
||||
self.touch()
|
||||
|
||||
|
||||
def pval_interpretation(p_val):
|
||||
if 0.005 >= p_val:
|
||||
return 'Diff'
|
||||
elif 0.05 >= p_val > 0.005:
|
||||
return 'Sim'
|
||||
elif p_val > 0.05:
|
||||
return 'Same'
|
||||
|
||||
|
||||
def color_red2green_01(val, maxtone=50):
|
||||
if np.isnan(val): return None
|
||||
assert 0 <= val <= 1, f'val {val} out of range [0,1]'
|
||||
|
||||
|
||||
# rescale to [-1,1]
|
||||
val = val * 2 - 1
|
||||
if val < 0:
|
||||
color = 'red'
|
||||
tone = maxtone * (-val)
|
||||
else:
|
||||
color = 'green'
|
||||
tone = maxtone * val
|
||||
return '\cellcolor{' + color + f'!{int(tone)}' + '}'
|
||||
|
|
@ -1,19 +1,17 @@
|
|||
import pickle
|
||||
import os
|
||||
|
||||
dataset = 'lequa/T1B'
|
||||
dataset = 'lequa/T1A'
|
||||
for metric in ['mae', 'mrae']:
|
||||
print('metric', metric)
|
||||
for method in ['KDEy-DMhd4', 'KDEy-DMhd4+', 'KDEy-DMhd4++']:
|
||||
|
||||
path = f'/home/moreo/QuaPy/distribution_matching/results/{dataset}/{metric}/{method}.hyper.pkl'
|
||||
if os.path.exists(path):
|
||||
obj = pickle.load(open(path, 'rb'))
|
||||
print(method, obj)
|
||||
else:
|
||||
print(f'path {path} does not exist')
|
||||
|
||||
print()
|
||||
method1 = 'KDEy-closed++'
|
||||
# method2 = 'KDEy-DMhd3+'
|
||||
# method1 = 'KDEy-ML'
|
||||
# method2 = 'KDEy-ML+'
|
||||
|
||||
path = f'../results/{dataset}/{metric}/{method1}.hyper.pkl'
|
||||
obj = pickle.load(open(path, 'rb'))
|
||||
print(method1, obj)
|
||||
|
||||
# path = f'../results/{dataset}/{metric}/{method2}.hyper.pkl'
|
||||
# obj = pickle.load(open(path, 'rb'))
|
||||
# print(method2, obj)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ if __name__ == '__main__':
|
|||
for method in METHODS:
|
||||
|
||||
print('Init method', method)
|
||||
if method == 'EMQ-C': continue
|
||||
|
||||
global_result_path = f'{result_dir}/{method}'
|
||||
|
||||
|
|
|
|||
|
|
@ -224,7 +224,7 @@ def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfol
|
|||
for train, test in data.kFCV(nfolds=nfolds, random_state=random_state):
|
||||
quantifier.fit(train)
|
||||
fold_prev = quantifier.quantify(test.X)
|
||||
rel_size = len(test)/len(data)
|
||||
rel_size = len(test.X)/len(data)
|
||||
total_prev += fold_prev*rel_size
|
||||
|
||||
return total_prev
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import quapy as qp
|
|||
|
||||
plt.rcParams['figure.figsize'] = [10, 6]
|
||||
plt.rcParams['figure.dpi'] = 200
|
||||
plt.rcParams['font.size'] = 12
|
||||
plt.rcParams['font.size'] = 18
|
||||
|
||||
|
||||
def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
|
||||
|
|
@ -259,7 +259,7 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
|
|||
data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order)
|
||||
|
||||
if method_order is None:
|
||||
method_order = np.unique(method_names)
|
||||
method_order = method_names
|
||||
|
||||
_set_colors(ax, n_methods=len(method_order))
|
||||
|
||||
|
|
@ -330,8 +330,8 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
|
|||
|
||||
if show_legend:
|
||||
fig.legend(loc='lower center',
|
||||
bbox_to_anchor=(0.9, 0.2),
|
||||
ncol=1)
|
||||
bbox_to_anchor=(1, 0.5),
|
||||
ncol=(len(method_names)+1)//2)
|
||||
|
||||
_save_or_show(savepath)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue