cleaning project

distribution_matching/commons.py

@@ -6,9 +6,14 @@ from distribution_matching.method_kdey_closed_efficient_correct import KDEyclose
 from quapy.method.aggregative import EMQ, CC, PCC, DistributionMatching, PACC, HDy, OneVsAllAggregative, ACC
 from distribution_matching.method_dirichlety import DIRy
 from sklearn.linear_model import LogisticRegression
-from method_kdey_closed_efficient import KDEyclosed_efficient
+from distribution_matching.method_kdey_closed_efficient import KDEyclosed_efficient
+
+# the full list of methods tested in the paper (reported in the appendix)
+METHODS  = ['ACC', 'PACC', 'HDy-OvA', 'DM-T', 'DM-HD', 'KDEy-HD', 'DM-CS', 'KDEy-CS',  'DIR', 'EMQ', 'EMQ-BCTS', 'KDEy-ML']
+
+# uncomment this other list for the methods shown in the body of the paper (the other methods are not comparable in performance)
+#METHODS  = ['PACC',  'DM-T', 'DM-HD', 'KDEy-HD', 'DM-CS', 'KDEy-CS',  'EMQ', 'KDEy-ML']
 
-METHODS  = ['ACC', 'PACC', 'HDy-OvA', 'DM-T', 'DM-HD', 'KDEy-DMhd4', 'DM-CS', 'KDEy-closed++',  'DIR', 'EMQ', 'KDEy-ML'] #['ACC', 'PACC', 'HDy-OvA', 'DIR', 'DM', 'KDEy-DMhd3', 'KDEy-closed++', 'EMQ', 'KDEy-ML'] #, 'KDEy-DMhd2'] #, 'KDEy-DMhd2', 'DM-HD'] 'KDEy-DMjs', 'KDEy-DM', 'KDEy-ML+', 'KDEy-DMhd3+', 'EMQ-C',
 BIN_METHODS = [x.replace('-OvA', '') for x in METHODS]
 
 
@@ -17,6 +22,12 @@ hyper_LR = {
     'classifier__class_weight': ['balanced', None]
 }
 
+hyper_kde = {
+    'bandwidth': np.linspace(0.01, 0.2, 20)
+}
+
+nbins_range = [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 64]
+
 def new_method(method, **lr_kwargs):
 
     lr = LogisticRegression(**lr_kwargs)
@@ -33,33 +44,22 @@ def new_method(method, **lr_kwargs):
     elif method == 'PACC':
         param_grid = hyper_LR
         quantifier = PACC(lr)
-    elif method == 'KDEy-ML':
+    elif method in ['KDEy-HD']:
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
+        param_grid = {**hyper_kde, **hyper_LR}
-        param_grid = {**method_params, **hyper_LR}
+        quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=10000, val_split=10)
-        quantifier = KDEy(lr, target='max_likelihood', val_split=10)
+    elif method == 'KDEy-CS':
-    elif method == 'KDEy-closed':
+        param_grid = {**hyper_kde, **hyper_LR}
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEyclosed(lr, val_split=10)
-    elif method == 'KDEy-closed+':
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEyclosed_efficient(lr, val_split=10)
-    elif method == 'KDEy-closed++':
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
         quantifier = KDEyclosed_efficient_corr(lr, val_split=10)
-    elif method in ['KDEy-DM']:
+    elif method == 'KDEy-ML':
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
+        param_grid = {**hyper_kde, **hyper_LR}
-        param_grid = {**method_params, **hyper_LR}
+        quantifier = KDEy(lr, target='max_likelihood', val_split=10)
-        quantifier = KDEy(lr, target='min_divergence', divergence='l2', montecarlo_trials=5000, val_split=10)
     elif method == 'DIR':
         param_grid = hyper_LR
         quantifier = DIRy(lr)
     elif method == 'EMQ':
         param_grid = hyper_LR
         quantifier = EMQ(lr)
-    elif method == 'EMQ-C':
+    elif method == 'EMQ-BCTS':
         method_params = {'exact_train_prev': [False], 'recalib': ['bcts']}
         param_grid = {**method_params, **hyper_LR}
         quantifier = EMQ(lr)
@@ -69,17 +69,9 @@ def new_method(method, **lr_kwargs):
     elif method == 'HDy-OvA':
         param_grid = {'binary_quantifier__' + key: val for key, val in hyper_LR.items()}
         quantifier = OneVsAllAggregative(HDy(lr))
-    elif method == 'DM':
-        method_params = {
-            'nbins': [4,8,16,32],
-            'val_split': [10, 0.4],
-            'divergence': ['HD', 'topsoe', 'l2']
-        }
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = DistributionMatching(lr)
     elif method == 'DM-T':
         method_params = {
-            'nbins': [2,3,4,5,6,7,8,9,10,12,14,16,18,20,22,24,26,28,30,32,64],
+            'nbins': nbins_range,
             'val_split': [10],
             'divergence': ['topsoe']
         }
@@ -87,7 +79,7 @@ def new_method(method, **lr_kwargs):
         quantifier = DistributionMatching(lr)
     elif method == 'DM-HD':
         method_params = {
-            'nbins': [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 64],
+            'nbins': nbins_range,
             'val_split': [10],
             'divergence': ['HD']
         }
@@ -95,55 +87,12 @@ def new_method(method, **lr_kwargs):
         quantifier = DistributionMatching(lr)
     elif method == 'DM-CS':
         method_params = {
-            'nbins': [2,3,4,5,6,7,8,9,10,12,14,16,18,20,22,24,26,28,30,32,64],
+            'nbins': nbins_range,
             'val_split': [10],
             'divergence': ['CS']
         }
         param_grid = {**method_params, **hyper_LR}
         quantifier = DistributionMatching(lr)
-
-    # experimental
-    elif method in ['KDEy-DMkld']:
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEy(lr, target='min_divergence', divergence='KLD', montecarlo_trials=5000, val_split=10)
-    # elif method in ['KDEy-DMhd']:
-    #     The code to reproduce this run is commented in the min_divergence target, I think it was incorrect...
-    #     method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-    #     param_grid = {**method_params, **hyper_LR}
-    #     quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
-    elif method in ['KDEy-DMhd2']:
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEy(lr, target='min_divergence_uniform', divergence='HD', montecarlo_trials=5000, val_split=10)
-    elif method in ['KDEy-DMjs']:
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEy(lr, target='min_divergence_uniform', divergence='JS', montecarlo_trials=5000, val_split=10)
-    elif method in ['KDEy-DMhd3']:
-        # I have realized that there was an error. I am sampling from the validation distribution (V) and not from the
-        # test distribution (T) just because the validation can be sampled in fit only once and pre-computed densities
-        # can be stored. This means that the reference distribution is V and not T. Then I have found that an
-        # f-divergence is defined as D(p||q) \int_{R^n}q(x)f(p(x)/q(x))dx = E_{x~q}[f(p(x)/q(x))], so if I am sampling
-        # V then I am computing D(T||V) (and not D(V||T) as I thought).
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
-    elif method in ['KDEy-DMhd4']:
-        # This is the new version in which we apply importance sampling, i.e., we compute:
-        #   D(p_a||q) = 1/N sum_x f(p(x)/q(x)) * (q(x)/r(x))
-        # where x ~iid r, with r = p_u, and u = (1/n, 1/n, ..., 1/n) the uniform vector
-        method_params = {'bandwidth': np.linspace(0.01, 0.2, 20)}
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = KDEy(lr, target='min_divergence', divergence='HD', montecarlo_trials=5000, val_split=10)
-    elif method == 'DM-HD':
-        method_params = {
-            'nbins': [4,8,16,32],
-            'val_split': [10, 0.4],
-        }
-        param_grid = {**method_params, **hyper_LR}
-        quantifier = DistributionMatching(lr, divergence='HD')
-
     else:
         raise NotImplementedError('unknown method', method)
 

distribution_matching/figures/histograms_density_plot.py

@@ -53,7 +53,7 @@ for i, post_set in enumerate([post_c1, post_c2, post_c3, post_test]):
     ax = fig.add_subplot(141+i, projection='3d')
     for post, c, z in zip(post_set.T, colors, positions):
 
-        hist, bins = np.histogram(post, bins=nbins, density=True)
+        hist, bins = np.histogram(post, bins=nbins, density=True, range=[0,1])
         xs = (bins[:-1] + bins[1:])/2
 
         ax.bar(xs, hist, width=1/nbins, zs=z, zdir='y', color=c, ec=c, alpha=0.6)

distribution_matching/method_kdey.py

@@ -129,7 +129,8 @@ class KDEy(AggregativeProbabilisticQuantifier):
         elif self.target == 'min_divergence':
             N = self.montecarlo_trials
             rs = self.random_state
-            self.reference_samples = np.vstack([kde_i.sample(N, random_state=rs) for kde_i in self.val_densities])
+            n = data.n_classes
+            self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.val_densities])
             self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.val_densities])
             self.reference_density = np.mean(self.reference_classwise_densities, axis=0)  # equiv. to (uniform @ self.reference_classwise_densities)
         elif self.target == 'min_divergence_deprecated':  # the version of the first draft, with n*N presampled, then alpha*N chosen for class
@@ -170,15 +171,48 @@ class KDEy(AggregativeProbabilisticQuantifier):
         else:
             raise ValueError('only squared HD is currently implemented')
 
+        epsilon = 1e-10
+        qs = test_densities + epsilon
+        rs = self.reference_density + epsilon
+        iw = qs/rs  #importance weights
+        p_class = self.reference_classwise_densities + epsilon
+        fracs = p_class/qs
+
+        def divergence(prev):
+            # ps / qs = (prev @ p_class) / qs = prev @ (p_class / qs) = prev @ fracs
+            ps_div_qs = prev @ fracs
+            return np.mean( f(ps_div_qs) * iw )
+
+        return F.optim_minimize(divergence, n_classes)
+
+    # new version in which we retain all n*N examples (sampled from a mixture with uniform parameter), and then
+    # apply importance sampling (IS). In this version we compute D(q||p_alpha) with IS, and not D(p_alpha||q) as
+    # in the reformulation proposed above
+    def _target_divergence_q__p(self, posteriors):
+        # in this variant we evaluate the divergence using a Montecarlo approach
+        n_classes = len(self.val_densities)
+
+        test_kde = self.get_kde_function(posteriors)
+        test_densities = self.pdf(test_kde, self.reference_samples)
+
+        def f_squared_hellinger(u):
+            return (np.sqrt(u)-1)**2
+
+        # todo: this will fail when self.divergence is a callable, and is not the right place to do it anyway
+        if self.divergence.lower() == 'hd':
+            f = f_squared_hellinger
+        else:
+            raise ValueError('only squared HD is currently implemented')
 
         epsilon = 1e-10
         qs = test_densities + epsilon
         rs = self.reference_density + epsilon
+        p_class = self.reference_classwise_densities + epsilon
 
-
+        # D(q||p_a) = 1/N sum f(q/p_a) * (p_a / p_u)
         def divergence(prev):
-            ps = prev @ self.reference_classwise_densities + epsilon
+            p_a = prev @ p_class
-            return np.mean( f(ps/qs) * (qs/rs) )
+            return np.mean( f(qs/p_a) * (p_a/rs) )
 
         return F.optim_minimize(divergence, n_classes)
 

distribution_matching/tables/gen_tables_compact.py

@@ -0,0 +1,249 @@
+from distribution_matching.commons import BIN_METHODS, METHODS
+import quapy as qp
+from os import makedirs
+import os
+
+from tabular import Table
+import pandas as pd
+
+tables_path = '.'
+# makedirs(tables_path, exist_ok=True)
+
+MAXTONE = 35  # sets the intensity of the maximum color reached by the worst (red) and best (green) results
+SHOW_STD = False
+
+NUM_ADJUSTMENT_METHODS = 2 if 'ACC' in METHODS else 1
+NUM_MAXIMUM_LIKELIHOOD_METHODS = 4 if 'DIR' in METHODS else 3
+NUM_DISTRIBUTION_MATCHING_PAIRS = 2
+NUM_DISTRIBUTION_MATCHING_METHODS = NUM_DISTRIBUTION_MATCHING_PAIRS*2 + (2 if 'HDy-OvA' in METHODS else 1)
+
+qp.environ['SAMPLE_SIZE'] = 100
+
+nice_bench = {
+    'sanders': 'Sanders',
+    'semeval13': 'SemEval13',
+    'semeval14': 'SemEval14',
+    'semeval15': 'SemEval15',
+    'semeval16': 'SemEval16',
+}
+
+nice_method={
+    'KDEy-MLE': 'KDEy-ML',
+    'KDEy-DMhd4': 'KDEy-HD',
+    'KDEy-closed++': 'KDEy-CS',
+    'EMQ-C': 'EMQ-BCTS'
+}
+
+def save_table(path, table):
+    print(f'saving results in {path}')
+    with open(path, 'wt') as foo:
+        foo.write(table)
+
+
+def nicerm(key):
+    return '\mathrm{'+nice[key]+'}'
+
+
+def make_table(tabs, eval, benchmark_groups, benchmark_names, compact=False):
+
+    n_methods = len(METHODS)
+    assert n_methods == (NUM_ADJUSTMENT_METHODS+NUM_DISTRIBUTION_MATCHING_METHODS+NUM_MAXIMUM_LIKELIHOOD_METHODS), \
+        "Unexpected number of methods"
+
+    cline = "\cline{2-" + str(n_methods+ 1) + "}"
+
+    # write the latex table
+    tabular = """
+            \\begin{tabular}{|c|""" + ('c|' * NUM_ADJUSTMENT_METHODS) + 'c|c' + ('|c|c' * (NUM_DISTRIBUTION_MATCHING_PAIRS)) +  ('|c' * NUM_MAXIMUM_LIKELIHOOD_METHODS) + """|} """ + cline + """           
+            \multicolumn{1}{c}{} & 
+            \multicolumn{"""+str(NUM_ADJUSTMENT_METHODS)+"""}{|c}{Adjustment} & 
+            \multicolumn{"""+str(NUM_DISTRIBUTION_MATCHING_METHODS)+"""}{|c|}{Distribution Matching} & 
+            \multicolumn{"""+str(NUM_MAXIMUM_LIKELIHOOD_METHODS)+"""}{c|}{Maximum Likelihood} \\\\
+            \hline               
+            """
+    for i, (tab, group, name) in enumerate(zip(tabs, benchmark_groups, benchmark_names)):
+        tablines = tab.latexTabular(benchmark_replace=nice_bench, method_replace=nice_method, endl='\\\\'+ cline, aslines=True)
+        print(tablines)
+        tablines[0] = tablines[0].replace('\multicolumn{1}{c|}{}', '\\textbf{'+name+'}')
+        if not compact:
+            tabular += '\n'.join(tablines)
+        else:
+            # if compact, keep the method names and the average; discard the rest
+            tabular += tablines[0] + '\n' + tablines[-1] + '\n'
+
+        tabular += "\n" + "\\textit{Rank} & " + tab.getRankTable(prec_mean=0 if name.startswith('LeQua') else 1).latexAverage()
+        if i < (len(tabs) - 1):
+            tabular += "\\hline\n"
+        else:
+            tabular += "\n"
+    tabular += "\end{tabular}"
+    return tabular
+
+
+def gen_tables_uci_multiclass(eval):
+
+    print('Generating table for UCI Multiclass Datasets', eval)
+    dir_results = f'../results/ucimulti/{eval}'
+
+    datasets = qp.datasets.UCI_MULTICLASS_DATASETS
+
+    tab = Table(
+        benchmarks=datasets,
+        methods=METHODS,
+        ttest='wilcoxon',
+        prec_mean=4,
+        show_std=SHOW_STD,
+        prec_std=4,
+        clean_zero=(eval=='mae'),
+        average=True,
+        maxtone=MAXTONE
+    )
+
+    for dataset in datasets:
+        print(f'\t Dataset: {dataset}: ', end='')
+        for method in METHODS:
+            result_path = f'{dir_results}/{method}_{dataset}.dataframe'
+            if os.path.exists(result_path):
+                df = pd.read_csv(result_path)
+                print(f'{method}', end=' ')
+                tab.add(dataset, method, df[eval].values)
+            else:
+                print(f'MISSING-{method}', end=' ')
+        print()
+
+    return tab
+
+
+def gen_tables_uci_bin(eval):
+
+    print('Generating table for UCI Datasets', eval)
+    dir_results = f'../results/binary/{eval}'
+
+    exclude = ['acute.a', 'acute.b', 'iris.1', 'balance.2']
+    datasets = [x for x in qp.datasets.UCI_DATASETS if x not in exclude]
+
+    tab = Table(
+        benchmarks=datasets,
+        methods=BIN_METHODS,
+        ttest='wilcoxon',
+        prec_mean=4,
+        show_std=SHOW_STD,
+        prec_std=4,
+        clean_zero=(eval=='mae'),
+        average=True,
+        maxtone=MAXTONE
+    )
+
+    for dataset in datasets:
+        print(f'\t Dataset: {dataset}: ', end='')
+        for method in BIN_METHODS:
+            result_path = f'{dir_results}/{method}_{dataset}.dataframe'
+            if os.path.exists(result_path):
+                df = pd.read_csv(result_path)
+                print(f'{method}', end=' ')
+                tab.add(dataset, method, df[eval].values)
+            else:
+                print(f'MISSING-{method}', end=' ')
+
+    return tab
+
+    
+
+def gen_tables_tweet(eval):
+
+    print('Generating table for Twitter', eval)
+    dir_results = f'../results/tweet/{eval}'
+
+    datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST
+
+    tab = Table(
+        benchmarks=datasets,
+        methods=METHODS,
+        ttest='wilcoxon',
+        prec_mean=4,
+        show_std=SHOW_STD,
+        prec_std=4,
+        clean_zero=(eval=='mae'),
+        average=True,
+        maxtone=MAXTONE
+    )
+
+    for dataset in datasets:
+        print(f'\t Dataset: {dataset}: ', end='')
+        for method in METHODS:
+            result_path = f'{dir_results}/{method}_{dataset}.dataframe'
+            if os.path.exists(result_path):
+                df = pd.read_csv(result_path)
+                print(f'{method}', end=' ')
+                tab.add(dataset, method, df[eval].values)
+            else:
+                print(f'MISSING-{method}', end=' ')
+        print()
+
+    return tab
+
+
+def gen_tables_lequa(Methods, task, eval):
+    # generating table for LeQua-T1A or Lequa-T1B; only one table with two rows, one for MAE, another for MRAE
+    dataset_name = 'LeQua-'+task
+
+    tab = Table(
+        benchmarks=[f'Average'],
+        methods=Methods,
+        ttest='wilcoxon',
+        prec_mean=5,
+        show_std=SHOW_STD,
+        prec_std=4,
+        clean_zero=False,
+        average=False,
+        maxtone=MAXTONE
+    )
+
+    print('Generating table for T1A@Lequa', eval, end='')
+    dir_results = f'../results/lequa/{task}/{eval}'
+
+    for method in Methods:
+        result_path = f'{dir_results}/{method}.dataframe'
+        if os.path.exists(result_path):
+            df = pd.read_csv(result_path)
+            print(f'{method}', end=' ')
+            tab.add('Average', method, df[eval].values)
+        else:
+            print(f'MISSING-{method}', end=' ')
+    print()
+
+    return tab
+
+
+
+if __name__ == '__main__':
+    os.makedirs('./latex', exist_ok=True)
+
+    for eval in ['mae', 'mrae']:
+        tabs = []
+        tabs.append(gen_tables_tweet(eval))
+        tabs.append(gen_tables_uci_multiclass(eval))
+        tabs.append(gen_tables_lequa(METHODS, 'T1B', eval))
+
+        names = ['Tweets', 'UCI-multi', 'LeQua-T1B']
+        table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=names)
+        save_table(f'./latex/multiclass_{eval}.tex', table)
+
+    for eval in ['mae', 'mrae']:
+        tabs = []
+        tabs.append(gen_tables_uci_bin(eval))
+        
+        # print uci-binary with all datasets for the appendix
+        table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=['UCI-binary'])
+        save_table(f'./latex/ucibinary_{eval}.tex', table)
+        
+        # print uci-bin compacted plus lequa-T1A for the main body
+        tabs.append(gen_tables_lequa(BIN_METHODS, 'T1A', eval))
+        table = make_table(tabs, eval, benchmark_groups=tabs, benchmark_names=['UCI-binary', 'LeQua-T1A'], compact=True)
+        save_table(f'./latex/binary_{eval}.tex', table)
+
+    print("[Tables Done] runing latex")
+    os.chdir('./latex/')
+    os.system('pdflatex tables_compact.tex')
+    os.system('rm tables_compact.aux tables_compact.bbl tables_compact.blg tables_compact.log tables_compact.out tables_compact.dvi')
+

distribution_matching/tables/latex/tables_compact.tex

@@ -0,0 +1,107 @@
+\documentclass{article}
+
+
+
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
+\usepackage{hyperref}       % hyperlinks
+\usepackage{url}            % simple URL typesetting
+\usepackage{booktabs}       % professional-quality tables
+\usepackage{amsfonts}       % blackboard math symbols
+\usepackage{nicefrac}       % compact symbols for 1/2, etc.
+\usepackage{microtype}      % microtypography
+\usepackage{lipsum}
+\usepackage{fancyhdr}       % header
+\usepackage{graphicx}       % graphics
+\graphicspath{{media/}}     % organize your images and other figures under media/ folder
+\usepackage{amsmath}
+\usepackage{bm}
+\usepackage{tabularx}
+\usepackage{color}
+\usepackage{colortbl}
+\usepackage{xcolor}
+\usepackage{lmodern}
+
+\DeclareMathOperator*{\argmax}{arg\,max}
+\DeclareMathOperator*{\argmin}{arg\,min}
+
+\newif\ifdraft
+\drafttrue
+
+\newcommand{\juanjo}[1]{\ifdraft{\leavevmode\color{purple}{[JJ]:
+{#1}}}\else{\vspace{0ex}}\fi}
+
+\newcommand{\alex}[1]{\ifdraft{\leavevmode\color{violet}{[AM]:
+{#1}}}\else{\vspace{0ex}}\fi}
+
+\newcommand{\pablo}[1]{\ifdraft{\leavevmode\color{red}{[PG]:
+{#1}}}\else{\vspace{0ex}}\fi}
+
+
+\title{Tables} 
+
+
+\author{
+  Alejandro Moreo 
+}
+
+
+\begin{document}
+
+\maketitle
+
+
+
+\begin{table}[h]
+ \centering
+ \caption{Multiclass AE}
+\resizebox{\textwidth}{!}{%
+\input{multiclass_mae}
+}%
+\end{table}
+
+
+\begin{table}[h]
+ \centering
+ \caption{Multiclass RAE}
+\resizebox{\textwidth}{!}{%
+\input{multiclass_mae}
+}%
+\end{table}
+
+\begin{table}[h]
+ \centering
+ \caption{Binary MAE}
+\resizebox{\textwidth}{!}{%
+\input{binary_mae}
+}%
+\end{table}
+
+\begin{table}[h]
+  \centering
+  \caption{Binary MRAE}
+ \resizebox{\textwidth}{!}{%
+ \input{binary_mrae}
+ }%
+ \end{table}
+
+\begin{table}[h]
+ \centering
+ \caption{UCI binary full AE}
+\resizebox{\textwidth}{!}{%
+\input{ucibinary_mae}
+}%
+\end{table}
+
+\begin{table}[h]
+ \centering
+ \caption{UCI binary full RAE}
+\resizebox{\textwidth}{!}{%
+\input{ucibinary_mrae}
+}%
+\end{table}
+
+
+
+\end{document}
+

distribution_matching/tables/tabular.py

@@ -0,0 +1,348 @@
+import numpy as np
+import itertools
+from scipy.stats import ttest_ind_from_stats, wilcoxon
+
+
+class Table:
+    VALID_TESTS = [None, "wilcoxon", "ttest"]
+
+    def __init__(self, benchmarks, methods, lower_is_better=True, ttest='ttest', prec_mean=3,
+                 clean_zero=False, show_std=False, prec_std=3, average=True, missing=None, missing_str='--', color=True, maxtone=50):
+        assert ttest in self.VALID_TESTS, f'unknown test, valid are {self.VALID_TESTS}'
+
+        self.benchmarks = np.asarray(benchmarks)
+        self.benchmark_index = {row:i for i, row in enumerate(benchmarks)}
+
+        self.methods = np.asarray(methods)
+        self.method_index = {col:j for j, col in enumerate(methods)}
+
+        self.map = {}  
+        # keyed (#rows,#cols)-ndarrays holding computations from self.map['values']
+        self._addmap('values', dtype=object)
+        self.lower_is_better = lower_is_better
+        self.ttest = ttest
+        self.prec_mean = prec_mean
+        self.clean_zero = clean_zero
+        self.show_std = show_std
+        self.prec_std = prec_std
+        self.add_average = average
+        self.missing = missing
+        self.missing_str = missing_str
+        self.color = color
+        self.maxtone = maxtone
+        
+        self.touch()
+
+    @property
+    def nbenchmarks(self):
+        return len(self.benchmarks)
+
+    @property
+    def nmethods(self):
+        return len(self.methods)
+
+    def touch(self):
+        self._modif = True
+
+    def update(self):
+        if self._modif:
+            self.compute()
+
+    def _getfilled(self):
+        return np.argwhere(self.map['fill'])
+
+    @property
+    def values(self):
+        return self.map['values']
+
+    def _indexes(self):
+        return itertools.product(range(self.nbenchmarks), range(self.nmethods))
+
+    def _addmap(self, map, dtype, func=None):
+        self.map[map] = np.empty((self.nbenchmarks, self.nmethods), dtype=dtype)
+        if func is None:
+            return
+        m = self.map[map]
+        f = func
+        indexes = self._indexes() if map == 'fill' else self._getfilled()
+        for i, j in indexes:
+            m[i, j] = f(self.values[i, j])
+
+    def _addrank(self):
+        for i in range(self.nbenchmarks):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            ranked_cols_idx = filled_cols_idx[np.argsort(col_means)]
+            if not self.lower_is_better:
+                ranked_cols_idx = ranked_cols_idx[::-1]
+            self.map['rank'][i, ranked_cols_idx] = np.arange(1, len(filled_cols_idx)+1)
+            
+    def _addcolor(self):
+        for i in range(self.nbenchmarks):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            if filled_cols_idx.size==0:
+                continue
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            #col_means = [self.map['rank'][i, j] for j in filled_cols_idx]
+
+            minval = min(col_means)
+            maxval = max(col_means)
+
+            for col_idx in filled_cols_idx:
+                val = self.map['mean'][i,col_idx]
+                norm = (maxval - minval)
+                if norm > 0:
+                    normval = (val - minval) / norm
+                else:
+                    normval = 0.5
+
+                if self.lower_is_better:
+                    normval = 1 - normval
+
+                normval = np.clip(normval, 0,1)
+
+                self.map['color'][i, col_idx] = color_red2green_01(normval, self.maxtone)
+
+    def _run_ttest(self, row, col1, col2):
+        mean1 = self.map['mean'][row, col1]
+        std1 = self.map['std'][row, col1]
+        nobs1 = self.map['nobs'][row, col1]
+        mean2 = self.map['mean'][row, col2]
+        std2 = self.map['std'][row, col2]
+        nobs2 = self.map['nobs'][row, col2]
+        _, p_val = ttest_ind_from_stats(mean1, std1, nobs1, mean2, std2, nobs2)
+        return p_val
+
+    def _run_wilcoxon(self, row, col1, col2):
+        values1 = self.map['values'][row, col1]
+        values2 = self.map['values'][row, col2]
+        try:
+            _, p_val = wilcoxon(values1, values2)
+        except ValueError:
+            p_val = 0
+        return p_val
+
+    def _add_statistical_test(self):
+        if self.ttest is None:
+            return
+        self.some_similar = [False]*self.nmethods
+        for i in range(self.nbenchmarks):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            if len(filled_cols_idx) <= 1:
+                continue
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            best_pos = filled_cols_idx[np.argmin(col_means)]
+
+            for j in filled_cols_idx:
+                if j==best_pos:
+                    continue
+                if self.ttest == 'ttest':
+                    p_val = self._run_ttest(i, best_pos, j)
+                else:
+                    p_val = self._run_wilcoxon(i, best_pos, j)
+
+                pval_outcome = pval_interpretation(p_val)
+                self.map['ttest'][i, j] = pval_outcome
+                if pval_outcome != 'Diff':
+                    self.some_similar[j] = True
+
+    def compute(self):
+        self._addmap('fill', dtype=bool, func=lambda x: x is not None)
+        self._addmap('mean', dtype=float, func=np.mean)
+        self._addmap('std', dtype=float, func=np.std)
+        self._addmap('nobs', dtype=float, func=len)
+        self._addmap('rank', dtype=int, func=None)
+        self._addmap('color', dtype=object, func=None)
+        self._addmap('ttest', dtype=object, func=None)
+        self._addmap('latex', dtype=object, func=None)
+        self._addrank()
+        self._addcolor()
+        self._add_statistical_test()
+        if self.add_average:
+            self._addave()
+        self._modif = False
+
+    def _is_column_full(self, col):
+        return all(self.map['fill'][:, self.method_index[col]])
+
+    def _addave(self):
+        ave = Table(['ave'], self.methods,
+                    lower_is_better=self.lower_is_better,
+                    ttest=self.ttest,
+                    average=False,
+                    missing=self.missing,
+                    missing_str=self.missing_str,
+                    prec_mean=self.prec_mean,
+                    prec_std=self.prec_std,
+                    clean_zero=self.clean_zero,
+                    show_std=self.show_std,
+                    color=self.color,
+                    maxtone=self.maxtone)
+        for col in self.methods:
+            values = None
+            if self._is_column_full(col):
+                if self.ttest == 'ttest':
+                    # values = np.asarray(self.map['mean'][:, self.method_index[col]])
+                    values = np.concatenate(self.values[:, self.method_index[col]])
+                else:  # wilcoxon
+                    # values = np.asarray(self.map['mean'][:, self.method_index[col]])
+                    values = np.concatenate(self.values[:, self.method_index[col]])
+            ave.add('ave', col, values)
+        self.average = ave
+
+    def add(self, benchmark, method, values):
+        if values is not None:
+            values = np.asarray(values)
+            if values.ndim==0:
+                values = values.flatten()
+        rid, cid = self._coordinates(benchmark, method)
+        self.map['values'][rid, cid] = values
+        self.touch()
+
+    def get(self, benchmark, method, attr='mean'):
+        self.update()
+        assert attr in self.map, f'unknwon attribute {attr}'
+        rid, cid = self._coordinates(benchmark, method)
+        if self.map['fill'][rid, cid]:
+            v = self.map[attr][rid, cid]
+            if v is None or (isinstance(v,float) and np.isnan(v)):
+                return self.missing
+            return v
+        else:
+            return self.missing
+
+    def _coordinates(self, benchmark, method):
+        assert benchmark in self.benchmark_index, f'benchmark {benchmark} out of range'
+        assert method in self.method_index, f'method {method} out of range'
+        rid = self.benchmark_index[benchmark]
+        cid = self.method_index[method]
+        return rid, cid
+
+    def get_average(self, method, attr='mean'):
+        self.update()
+        if self.add_average:
+            return self.average.get('ave', method, attr=attr)
+        return None
+
+    def get_color(self, benchmark, method):
+        color = self.get(benchmark, method, attr='color')
+        if color is None:
+            return ''
+        return color
+
+    def latex(self, benchmark, method):
+        self.update()
+        i,j = self._coordinates(benchmark, method)
+        if self.map['fill'][i,j] == False:
+            return self.missing_str
+
+        mean = self.map['mean'][i,j]
+        l = f" {mean:.{self.prec_mean}f}"
+        if self.clean_zero:
+            l = l.replace(' 0.', '.')
+
+        isbest = self.map['rank'][i,j] == 1
+        if isbest:
+            l = "\\textbf{"+l.strip()+"}"
+
+        stat = '' if self.ttest is None else '^{\phantom{\ddag}}'
+        if self.ttest is not None and self.some_similar[j]:
+            test_label = self.map['ttest'][i,j]
+            if test_label == 'Sim':
+                stat = '^{\dag}'
+            elif test_label == 'Same':
+                stat = '^{\ddag}'
+            elif isbest or test_label == 'Diff':
+                stat = '^{\phantom{\ddag}}'
+
+        std = ''
+        if self.show_std:
+            std = self.map['std'][i,j]
+            std = f" {std:.{self.prec_std}f}"
+            if self.clean_zero:
+                std = std.replace(' 0.', '.')
+            std = f"\pm {std:{self.prec_std}}"
+
+        if stat!='' or std!='':
+            l = f'{l}${stat}{std}$'
+
+        if self.color:
+            l += ' ' + self.map['color'][i,j]
+
+        return l
+
+    def latexTabular(self, benchmark_replace={}, method_replace={}, aslines=False, endl='\\\\\hline'):
+        lines = []
+        l = '\multicolumn{1}{c|}{} & '
+        l += ' & '.join([method_replace.get(col, col) for col in self.methods])
+        l += ' \\\\\hline'
+        lines.append(l)
+
+        for row in self.benchmarks:
+            rowname = benchmark_replace.get(row, row)
+            l = rowname + ' & '
+            l += self.latexRow(row, endl=endl)
+            lines.append(l)
+
+        if self.add_average:
+            # l += '\hline\n'
+            l = '\hline \n \\textit{Average} & '
+            l += self.latexAverage(endl=endl)
+            lines.append(l)
+        if not aslines:
+            lines='\n'.join(lines)
+        return lines
+
+    def latexRow(self, benchmark, endl='\\\\\hline\n'):
+        s = [self.latex(benchmark, col) for col in self.methods]
+        s = ' & '.join(s)
+        s += ' ' + endl
+        return s
+
+    def latexAverage(self, endl='\\\\\hline\n'):
+        if self.add_average:
+            return self.average.latexRow('ave', endl=endl)
+
+    def getRankTable(self, prec_mean=0):
+        t = Table(benchmarks=self.benchmarks, methods=self.methods, prec_mean=prec_mean, average=True, maxtone=self.maxtone, ttest=None)
+        for rid, cid in self._getfilled():
+            row = self.benchmarks[rid]
+            col = self.methods[cid]
+            t.add(row, col, self.get(row, col, 'rank'))
+        t.compute()
+        return t
+
+    def dropMethods(self, methods):
+        drop_index = [self.method_index[m] for m in methods]
+        new_methods = np.delete(self.methods, drop_index)
+        new_index = {col:j for j, col in enumerate(new_methods)}
+
+        self.map['values'] = self.values[:,np.asarray([self.method_index[m] for m in new_methods], dtype=int)]
+        self.methods = new_methods
+        self.method_index = new_index
+        self.touch()
+
+
+def pval_interpretation(p_val):
+    if 0.005 >= p_val:
+        return 'Diff'
+    elif 0.05 >= p_val > 0.005:
+        return 'Sim'
+    elif p_val > 0.05:
+        return 'Same'
+
+
+def color_red2green_01(val, maxtone=50):
+    if np.isnan(val): return None
+    assert 0 <= val <= 1, f'val {val} out of range [0,1]'
+
+
+    # rescale to [-1,1]
+    val = val * 2 - 1
+    if val < 0:
+        color = 'red'
+        tone = maxtone * (-val)
+    else:
+        color = 'green'
+        tone = maxtone * val
+    return '\cellcolor{' + color + f'!{int(tone)}' + '}'

distribution_matching/tmp/check_hyperparams.py

@@ -1,17 +1,19 @@
 import pickle
+import os
 
-dataset = 'lequa/T1A'
+dataset = 'lequa/T1B'
 for metric in ['mae', 'mrae']:
-    method1 = 'KDEy-closed++'
+    print('metric', metric)
-    # method2 = 'KDEy-DMhd3+'
+    for method in ['KDEy-DMhd4', 'KDEy-DMhd4+', 'KDEy-DMhd4++']:
-    # method1 = 'KDEy-ML'
-    # method2 = 'KDEy-ML+'
 
-    path = f'../results/{dataset}/{metric}/{method1}.hyper.pkl'
+        path = f'/home/moreo/QuaPy/distribution_matching/results/{dataset}/{metric}/{method}.hyper.pkl'
+        if os.path.exists(path):
             obj = pickle.load(open(path, 'rb'))
-    print(method1, obj)
+            print(method, obj)
+        else:
+            print(f'path {path} does not exist')
+
+    print()
+
 
-    # path = f'../results/{dataset}/{metric}/{method2}.hyper.pkl'
-    # obj = pickle.load(open(path, 'rb'))
-    # print(method2, obj)
 

distribution_matching/tweets_experiments.py

@@ -24,7 +24,6 @@ if __name__ == '__main__':
         for method in METHODS:
 
             print('Init method', method)
-            if method == 'EMQ-C': continue
 
             global_result_path = f'{result_dir}/{method}'
 

quapy/model_selection.py

@@ -224,7 +224,7 @@ def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfol
     for train, test in data.kFCV(nfolds=nfolds, random_state=random_state):
         quantifier.fit(train)
         fold_prev = quantifier.quantify(test.X)
-        rel_size = len(test.X)/len(data)
+        rel_size = len(test)/len(data)
         total_prev += fold_prev*rel_size
 
     return total_prev

quapy/plot.py

@@ -11,7 +11,7 @@ import quapy as qp
 
 plt.rcParams['figure.figsize'] = [10, 6]
 plt.rcParams['figure.dpi'] = 200
-plt.rcParams['font.size'] = 18
+plt.rcParams['font.size'] = 12
 
 
 def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
@@ -259,7 +259,7 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
     data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order)
 
     if method_order is None:
-        method_order = method_names
+        method_order = np.unique(method_names)
 
     _set_colors(ax, n_methods=len(method_order))
 
@@ -330,8 +330,8 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
     
     if show_legend:
         fig.legend(loc='lower center',
-                  bbox_to_anchor=(1, 0.5),
+                  bbox_to_anchor=(0.9, 0.2),
-                  ncol=(len(method_names)+1)//2)
+                  ncol=1)
       
     _save_or_show(savepath)