adding table manager

TweetSentQuant/experiments.py

@@ -1,5 +1,6 @@
 from sklearn.linear_model import LogisticRegression
 import quapy as qp
+from quapy.method.aggregative import OneVsAll
 import quapy.functional as F
 import numpy as np
 import os
@@ -7,9 +8,7 @@ import pickle
 import itertools
 from joblib import Parallel, delayed
 import multiprocessing
-
+import settings
-
-n_jobs = multiprocessing.cpu_count()
 
 
 def quantification_models():
@@ -17,11 +16,19 @@ def quantification_models():
         return LogisticRegression(max_iter=1000, solver='lbfgs', n_jobs=-1)
     __C_range = np.logspace(-4, 5, 10)
     lr_params = {'C': __C_range, 'class_weight': [None, 'balanced']}
+    svmperf_params = {'C': __C_range}
     yield 'cc', qp.method.aggregative.CC(newLR()), lr_params
     yield 'acc', qp.method.aggregative.ACC(newLR()), lr_params
     yield 'pcc', qp.method.aggregative.PCC(newLR()), lr_params
     yield 'pacc', qp.method.aggregative.PACC(newLR()), lr_params
     yield 'sld', qp.method.aggregative.EMQ(newLR()), lr_params
+    yield 'svmq', OneVsAll(qp.method.aggregative.SVMQ(settings.SVMPERF_HOME)), svmperf_params
+    yield 'svmkld', OneVsAll(qp.method.aggregative.SVMKLD(settings.SVMPERF_HOME)), svmperf_params
+    yield 'svmnkld', OneVsAll(qp.method.aggregative.SVMNKLD(settings.SVMPERF_HOME)), svmperf_params
+
+#     'svmmae': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='mae'),
+#     'svmmrae': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='mrae'),
+#     'mlpe': lambda learner: MaximumLikelihoodPrevalenceEstimation(),
 
 
 def evaluate_experiment(true_prevalences, estim_prevalences):
@@ -73,6 +80,7 @@ def run(experiment):
         print(f'running dataset={dataset_name} model={model_name} loss={optim_loss}')
 
     benchmark_devel = qp.datasets.fetch_twitter(dataset_name, for_model_selection=True, min_df=5, pickle=True)
+    benchmark_devel.stats()
 
     # model selection (hyperparameter optimization for a quantification-oriented loss)
     model_selection = qp.model_selection.GridSearchQ(
@@ -122,18 +130,8 @@ if __name__ == '__main__':
     datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TRAIN
     models = quantification_models()
 
-    results = Parallel(n_jobs=n_jobs)(
+    results = Parallel(n_jobs=settings.N_JOBS)(
         delayed(run)(experiment) for experiment in itertools.product(optim_losses, datasets, models)
     )
 
 
-# QUANTIFIER_ALIASES = {
-#     'emq': lambda learner: ExpectationMaximizationQuantifier(learner),
-#     'svmq': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='q'),
-#     'svmkld': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='kld'),
-#     'svmnkld': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='nkld'),
-#     'svmmae': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='mae'),
-#     'svmmrae': lambda learner: OneVsAllELM(settings.SVM_PERF_HOME, loss='mrae'),
-#     'mlpe': lambda learner: MaximumLikelihoodPrevalenceEstimation(),
-# }
-#

TweetSentQuant/settings.py

@@ -0,0 +1,5 @@
+import multiprocessing
+
+
+N_JOBS = -2  #multiprocessing.cpu_count()
+SVMPERF_HOME = '../svm_perf_quantification'

TweetSentQuant/tables2.py

@@ -0,0 +1,207 @@
+import quapy as qp
+import numpy as np
+from os import makedirs
+# from evaluate import evaluate_directory, statistical_significance, get_ranks_from_Gao_Sebastiani
+import sys, os
+import pickle
+from experiments import result_path
+from result_manager import ResultSet
+from tabular import Table
+
+tables_path = './tables'
+MAXTONE = 50  # sets the intensity of the maximum color reached by the worst (red) and best (green) results
+
+makedirs(tables_path, exist_ok=True)
+
+sample_size = 100
+qp.environ['SAMPLE_SIZE'] = sample_size
+
+
+nice = {
+    'mae':'AE',
+    'mrae':'RAE',
+    'ae':'AE',
+    'rae':'RAE',
+    'svmkld': 'SVM(KLD)',
+    'svmnkld': 'SVM(NKLD)',
+    'svmq': 'SVM(Q)',
+    'svmae': 'SVM(AE)',
+    'svmnae': 'SVM(NAE)',
+    'svmmae': 'SVM(AE)',
+    'svmmrae': 'SVM(RAE)',
+    'quanet': 'QuaNet',
+    'hdy': 'HDy',
+    'dys': 'DyS',
+    'svmperf':'',
+    'sanders': 'Sanders',
+    'semeval13': 'SemEval13',
+    'semeval14': 'SemEval14',
+    'semeval15': 'SemEval15',
+    'semeval16': 'SemEval16',
+    'Average': 'Average'
+}
+
+
+
+def nicerm(key):
+    return '\mathrm{'+nice[key]+'}'
+
+
+def load_Gao_Sebastiani_previous_results():
+    def rename(method):
+        old2new = {
+            'kld': 'svmkld',
+            'nkld': 'svmnkld',
+            'qbeta2': 'svmq',
+            'em': 'sld'
+        }
+        return old2new.get(method, method)
+
+    gao_seb_results = {}
+    with open('./Gao_Sebastiani_results.txt', 'rt') as fin:
+        lines = fin.readlines()
+        for line in lines[1:]:
+            line = line.strip()
+            parts = line.lower().split()
+            if len(parts) == 4:
+                dataset, method, ae, rae = parts
+            else:
+                method, ae, rae = parts
+            learner, method = method.split('-')
+            method = rename(method)
+            gao_seb_results[f'{dataset}-{method}-ae'] = float(ae)
+            gao_seb_results[f'{dataset}-{method}-rae'] = float(rae)
+    return gao_seb_results
+
+
+def get_ranks_from_Gao_Sebastiani():
+    gao_seb_results = load_Gao_Sebastiani_previous_results()
+    datasets = set([key.split('-')[0] for key in gao_seb_results.keys()])
+    methods = np.sort(np.unique([key.split('-')[1] for key in gao_seb_results.keys()]))
+    ranks = {}
+    for metric in ['ae', 'rae']:
+        for dataset in datasets:
+            scores = [gao_seb_results[f'{dataset}-{method}-{metric}'] for method in methods]
+            order = np.argsort(scores)
+            sorted_methods = methods[order]
+            for i, method in enumerate(sorted_methods):
+                ranks[f'{dataset}-{method}-{metric}'] = i+1
+        for method in methods:
+            rankave = np.mean([ranks[f'{dataset}-{method}-{metric}'] for dataset in datasets])
+            ranks[f'Average-{method}-{metric}'] = rankave
+    return ranks, gao_seb_results
+
+
+def save_table(path, table):
+    print(f'saving results in {path}')
+    with open(path, 'wt') as foo:
+        foo.write(table)
+
+
+
+datasets = qp.datasets.TWITTER_SENTIMENT_DATASETS_TEST
+evaluation_measures = [qp.error.ae, qp.error.rae]
+gao_seb_methods = ['cc', 'acc', 'pcc', 'pacc', 'sld', 'svmq', 'svmkld', 'svmnkld']
+new_methods = []
+
+def addfunc(dataset, method, loss):
+    path = result_path(dataset, method, 'm'+loss if not loss.startswith('m') else loss)
+    if os.path.exists(path):
+        true_prevs, estim_prevs, _, _, _, _ = pickle.load(open(path, 'rb'))
+        err_fn = getattr(qp.error, loss)
+        errors = err_fn(true_prevs, estim_prevs)
+        return errors
+    return None
+
+
+gao_seb_ranks, gao_seb_results = get_ranks_from_Gao_Sebastiani()
+
+for i, eval_func in enumerate(evaluation_measures):
+
+    # Tables evaluation scores for AE and RAE (two tables)
+    # ----------------------------------------------------
+
+    eval_name = eval_func.__name__
+    added_methods = ['svm' + eval_name] + new_methods
+    methods = gao_seb_methods + added_methods
+    nold_methods = len(gao_seb_methods)
+    nnew_methods = len(added_methods)
+
+    table = Table(rows=datasets, cols=methods, addfunc=addfunc)
+
+    # fill table
+    for dataset in datasets:
+        for method in methods:
+            table.add(dataset, method, eval_name)
+
+    tabular = """
+    \\begin{tabularx}{\\textwidth}{|c||""" + ('Y|'*len(gao_seb_methods))+ '|' + ('Y|'*len(added_methods)) + """} \hline
+      & \multicolumn{"""+str(nold_methods)+"""}{c||}{Methods tested in~\cite{Gao:2016uq}} & \multicolumn{"""+str(nnew_methods)+"""}{c|}{} \\\\ \hline
+    """
+
+    rowreplace={dataset: nice.get(dataset, dataset.upper()) for dataset in datasets}
+    colreplace={method:'\side{' + nice.get(method, method.upper()) +'$^{' + nicerm(eval_name) + '}$} ' for method in methods}
+
+    tabular += table.latextabular(rowreplace=rowreplace, colreplace=colreplace)
+    tabular += "\n\end{tabularx}"
+
+    save_table(f'./tables/tab_results_{eval_name}.new2.tex', tabular)
+
+
+
+    # Tables ranks for AE and RAE (two tables)
+    # ----------------------------------------------------
+    def addfuncRank(dataset, method):
+        rank = table.get(dataset, method, 'rank')
+        if rank is None:
+            return None
+        return [rank]
+
+    methods = gao_seb_methods
+    nold_methods = len(gao_seb_methods)
+
+    ranktable = Table(rows=datasets, cols=methods, addfunc=addfuncRank)
+    # fill table
+    for dataset in datasets:
+        for method in methods:
+            ranktable.add(dataset, method)
+
+
+    tabular = """
+    \\begin{tabularx}{\\textwidth}{|c||""" + ('Y|' * len(gao_seb_methods)) + """} \hline
+          & \multicolumn{""" + str(nold_methods) + """}{c||}{Methods tested in~\cite{Gao:2016uq}}  \\\\ \hline
+    """
+    for method in methods:
+        tabular += ' & \side{' + nice.get(method, method.upper()) +'$^{' + nicerm(eval_name) + '}$} '
+    tabular += '\\\\\hline\n'
+
+    for dataset in datasets:
+        tabular += nice.get(dataset, dataset.upper()) + ' '
+        for method in methods:
+            newrank = ranktable.get(dataset, method)
+            oldrank = gao_seb_ranks[f'{dataset}-{method}-{eval_name}']
+            if newrank is None:
+                newrank = '--'
+            else:
+                newrank =  f'{int(newrank)}'
+            tabular += ' & ' + f'{newrank}' + f' ({oldrank}) ' + ranktable.get_color(dataset, method)
+        tabular += '\\\\\hline\n'
+
+    tabular += 'Average & '
+    for method in methods:
+        newrank = ranktable.get_col_average(method)
+        oldrank = gao_seb_ranks[f'Average-{method}-{eval_name}']
+        if newrank is None or np.isnan(newrank):
+            newrank = '--'
+        else:
+            newrank = f'{newrank:.1f}'
+        oldrank = f'{oldrank:.1f}'
+        tabular += ' & ' + f'{newrank}' + f' ({oldrank}) ' + ranktable.get_color(dataset, method)
+    tabular += '\\\\\hline\n'
+
+    tabular += "\end{tabularx}"
+
+    save_table(f'./tables/tab_rank_{eval_name}.new2.tex', tabular)
+
+
+print("[Done]")

TweetSentQuant/tabular.py

@@ -0,0 +1,372 @@
+import numpy as np
+import itertools
+from scipy.stats import ttest_ind_from_stats, wilcoxon
+
+
+class Table:
+    VALID_TESTS = [None, "wilcoxon", "ttest"]
+
+    def __init__(self, rows, cols, addfunc, lower_is_better=True, ttest='ttest', prec_mean=3, clean_zero=False,
+                 show_std=False, prec_std=3):
+        assert ttest in self.VALID_TESTS, f'unknown test, valid are {self.VALID_TESTS}'
+        self.rows = np.asarray(rows)
+        self.row_index = {row:i for i,row in enumerate(rows)}
+        self.cols = np.asarray(cols)
+        self.col_index = {col:j for j,col in enumerate(cols)}
+        self.map = {}
+        self.mfunc = {}
+        self.rarr = {}
+        self.carr = {}
+        self._addmap('values', dtype=object)
+        self._addmap('fill', dtype=bool, func=lambda x: x is not None)
+        self._addmap('mean', dtype=float, func=np.mean)
+        self._addmap('std', dtype=float, func=np.std)
+        self._addmap('nobs', dtype=float, func=len)
+        self._addmap('rank', dtype=int, func=None)
+        self._addmap('color', dtype=object, func=None)
+        self._addmap('ttest', dtype=object, func=None)
+        self._addrarr('mean', dtype=float, func=np.mean, argmap='mean')
+        self._addrarr('min', dtype=float, func=np.min, argmap='mean')
+        self._addrarr('max', dtype=float, func=np.max, argmap='mean')
+        self._addcarr('mean', dtype=float, func=np.mean, argmap='mean')
+        self._addcarr('rank-mean', dtype=float, func=np.mean, argmap='rank')
+        if self.nrows>1:
+            self._col_ttest = Table(['ttest'], cols, _merge, lower_is_better, ttest)
+        else:
+            self._col_ttest = None
+        self.addfunc = addfunc
+        self.lower_is_better = lower_is_better
+        self.ttest = ttest
+        self.prec_mean = prec_mean
+        self.clean_zero = clean_zero
+        self.show_std = show_std
+        self.prec_std = prec_std
+        self.touch()
+
+    @property
+    def nrows(self):
+        return len(self.rows)
+
+    @property
+    def ncols(self):
+        return len(self.cols)
+
+    def touch(self):
+        self.modif = True
+
+    def update(self):
+        if self.modif:
+            self.compute()
+
+    def _addmap(self, map, dtype, func=None):
+        self.map[map] = np.empty((self.nrows, self.ncols), dtype=dtype)
+        self.mfunc[map] = func
+        self.touch()
+
+    def _addrarr(self, rarr, dtype, func=np.mean, argmap='mean'):
+        self.rarr[rarr] = {
+            'arr': np.empty(self.ncols, dtype=dtype),
+            'func': func,
+            'argmap': argmap
+        }
+        self.touch()
+
+    def _addcarr(self, carr, dtype, func=np.mean, argmap='mean'):
+        self.carr[carr] = {
+            'arr': np.empty(self.nrows, dtype=dtype),
+            'func': func,
+            'argmap': argmap
+        }
+        self.touch()
+
+    def _getfilled(self):
+        return np.argwhere(self.map['fill'])
+
+    @property
+    def values(self):
+        return self.map['values']
+
+    def _indexes(self):
+        return itertools.product(range(self.nrows), range(self.ncols))
+
+    def _runmap(self, map):
+        m = self.map[map]
+        f = self.mfunc[map]
+        if f is None:
+            return
+        indexes = self._indexes() if map == 'fill' else self._getfilled()
+        for i,j in indexes:
+            m[i,j] = f(self.values[i,j])
+
+    def _runrarr(self, rarr):
+        dic = self.rarr[rarr]
+        arr, f, map = dic['arr'], dic['func'], dic['argmap']
+        for col, cid in self.col_index.items():
+            if all(self.map['fill'][:, cid]):
+                arr[cid] = f(self.map[map][:, cid])
+            else:
+                arr[cid] = None
+
+    def _runcarr(self, carr):
+        dic = self.carr[carr]
+        arr, f, map = dic['arr'], dic['func'], dic['argmap']
+        for row, rid in self.row_index.items():
+            if all(self.map['fill'][rid, :]):
+                arr[rid] = f(self.map[map][rid, :])
+            else:
+                arr[rid] = None
+
+    def _runrank(self):
+        for i in range(self.nrows):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            ranked_cols_idx = filled_cols_idx[np.argsort(col_means)]
+            if not self.lower_is_better:
+                ranked_cols_idx = ranked_cols_idx[::-1]
+            self.map['rank'][i, ranked_cols_idx] = np.arange(1, len(filled_cols_idx)+1)
+
+    def _runcolor(self):
+        for i in range(self.nrows):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            if filled_cols_idx.size==0:
+                continue
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            minval = min(col_means)
+            maxval = max(col_means)
+            for col_idx in filled_cols_idx:
+                val = self.map['mean'][i,col_idx]
+                norm = (maxval - minval)
+                if norm > 0:
+                    normval = (val - minval) / norm
+                else:
+                    normval = 0.5
+                if self.lower_is_better:
+                    normval = 1 - normval
+                self.map['color'][i, col_idx] = color_red2green_01(normval)
+
+    def _run_ttest(self, row, col1, col2):
+        mean1 = self.map['mean'][row, col1]
+        std1 = self.map['std'][row, col1]
+        nobs1 = self.map['nobs'][row, col1]
+        mean2 = self.map['mean'][row, col2]
+        std2 = self.map['std'][row, col2]
+        nobs2 = self.map['nobs'][row, col2]
+        _, p_val = ttest_ind_from_stats(mean1, std1, nobs1, mean2, std2, nobs2)
+        return p_val
+
+    def _run_wilcoxon(self, row, col1, col2):
+        values1 = self.map['values'][row, col1]
+        values2 = self.map['values'][row, col2]
+        _, p_val = wilcoxon(values1, values2)
+        return p_val
+
+    def _runttest(self):
+        if self.ttest is None:
+            return
+        self.some_similar = False
+        for i in range(self.nrows):
+            filled_cols_idx = np.argwhere(self.map['fill'][i]).flatten()
+            if len(filled_cols_idx) <= 1:
+                continue
+            col_means = [self.map['mean'][i,j] for j in filled_cols_idx]
+            best_pos = filled_cols_idx[np.argmin(col_means)]
+
+            for j in filled_cols_idx:
+                if j==best_pos:
+                    continue
+                if self.ttest == 'ttest':
+                    p_val = self._run_ttest(i, best_pos, j)
+                else:
+                    p_val = self._run_wilcoxon(i, best_pos, j)
+
+                pval_outcome = pval_interpretation(p_val)
+                self.map['ttest'][i, j] = pval_outcome
+                if pval_outcome != 'Diff':
+                    self.some_similar = True
+
+    def get_col_average(self, col, arr='mean'):
+        self.update()
+        cid = self.col_index[col]
+        return self.rarr[arr]['arr'][cid]
+
+    def _map_list(self):
+        maps = list(self.map.keys())
+        maps.remove('fill')
+        maps.remove('values')
+        maps.remove('color')
+        maps.remove('ttest')
+        return ['fill'] + maps
+
+    def compute(self):
+        for map in self._map_list():
+            self._runmap(map)
+        self._runrank()
+        self._runcolor()
+        self._runttest()
+        for arr in self.rarr.keys():
+            self._runrarr(arr)
+        for arr in self.carr.keys():
+            self._runcarr(arr)
+        if self._col_ttest != None:
+            for col in self.cols:
+                self._col_ttest.add('ttest', col, self.col_index[col], self.map['fill'], self.values, self.map['mean'], self.ttest)
+                self._col_ttest.compute()
+        self.modif = False
+
+    def add(self, row, col, *args, **kwargs):
+        print(row, col, args, kwargs)
+        values = self.addfunc(row, col, *args, **kwargs)
+        # if values is None:
+        #     raise ValueError(f'addfunc returned None for row={row} col={col}')
+        rid, cid = self.coord(row, col)
+        self.map['values'][rid, cid] = values
+        self.touch()
+
+    def get(self, row, col, attr='mean'):
+        assert attr in self.map, f'unknwon attribute {attr}'
+        self.update()
+        rid, cid = self.coord(row, col)
+        if self.map['fill'][rid, cid]:
+            return self.map[attr][rid, cid]
+
+    def coord(self, row, col):
+        assert row in self.row_index, f'row {row} out of range'
+        assert col in self.col_index, f'col {col} out of range'
+        rid = self.row_index[row]
+        cid = self.col_index[col]
+        return rid, cid
+
+    def get_col_table(self):
+        return self._col_ttest
+
+    def get_color(self, row, col):
+        color = self.get(row, col, attr='color')
+        if color is None:
+            return ''
+        return color
+
+    def latex(self, row, col, missing='--', color=True):
+        self.update()
+        i,j = self.coord(row, col)
+        if self.map['fill'][i,j] == False:
+            return missing
+
+        mean = self.map['mean'][i,j]
+        l = f" {mean:.{self.prec_mean}f}"
+        if self.clean_zero:
+            l = l.replace(' 0.', '.')
+
+        isbest = self.map['rank'][i,j] == 1
+
+        if isbest:
+            l = "\\textbf{"+l+"}"
+        else:
+            if self.ttest is not None and self.some_similar:
+                test_label = self.map['ttest'][i,j]
+                if test_label == 'Sim':
+                    l += '^{\dag\phantom{\dag}}'
+                elif test_label == 'Same':
+                    l += '^{\ddag}'
+                elif test_label == 'Diff':
+                    l += '^{\phantom{\ddag}}'
+
+        if self.show_std:
+            std = self.map['std'][i,j]
+            std = f" {std:.{self.prec_std}f}"
+            if self.clean_zero:
+                std = std.replace(' 0.', '.')
+            l += f" \pm {std}"
+
+        l = f'$ {l} $'
+        if color:
+            l += ' ' + self.map['color'][i,j]
+
+        return l
+
+    def latextabular(self, missing='--', color=True, rowreplace={}, colreplace={}, average=True):
+        tab = ' & '
+        tab += ' & '.join([colreplace.get(col, col) for col in self.cols])
+        tab += ' \\\\\hline\n'
+        for row in self.rows:
+            rowname = rowreplace.get(row, row)
+            tab += rowname + ' & '
+            tab += self.latexrow(row, missing, color)
+            tab += ' \\\\\hline\n'
+
+        if average:
+            tab += 'Average & '
+            tab += self.latexave(missing, color)
+            tab += ' \\\\\hline\n'
+        return tab
+
+
+    def latexrow(self, row, missing='--', color=True):
+        s = [self.latex(row, col, missing=missing, color=color) for col in self.cols]
+        s = ' & '.join(s)
+        return s
+
+    def latexave(self, missing='--', color=True):
+        return self._col_ttest.latexrow('ttest')
+
+    def get_rank_table(self):
+        t = Table(rows=self.rows, cols=self.cols, addfunc=_getrank, ttest=None, prec_mean=0)
+        for row, col in self._getfilled():
+            t.add(self.rows[row], self.cols[col], row, col, self.map['rank'])
+        return t
+
+def _getrank(row, col, rowid, colid, rank):
+    return [rank[rowid, colid]]
+
+def _merge(unused, col, colidx, fill, values, means, ttest):
+    if all(fill[:,colidx]):
+        nrows = values.shape[0]
+        if ttest=='ttest':
+            values = np.asarray(means[:, colidx])
+        else:  # wilcoxon
+            values = [values[i, colidx] for i in range(nrows)]
+            values = np.concatenate(values)
+        return values
+    else:
+        return None
+
+def pval_interpretation(p_val):
+    if 0.005 >= p_val:
+        return 'Diff'
+    elif 0.05 >= p_val > 0.005:
+        return 'Sim'
+    elif p_val > 0.05:
+        return 'Same'
+
+
+def color_red2green_01(val, maxtone=50):
+    if np.isnan(val): return None
+    assert 0 <= val <= 1, f'val {val} out of range [0,1]'
+
+    # rescale to [-1,1]
+    val = val * 2 - 1
+    if val < 0:
+        color = 'red'
+        tone = maxtone * (-val)
+    else:
+        color = 'green'
+        tone = maxtone * val
+    return '\cellcolor{' + color + f'!{int(tone)}' + '}'
+
+#
+# def addfunc(m,d, mean, size):
+#     return np.random.rand(size)+mean
+#
+# t = Table(rows = ['M1', 'M2', 'M3'], cols=['D1', 'D2', 'D3', 'D4'], addfunc=addfunc, ttest='wilcoxon')
+# t.add('M1','D1', mean=0.5, size=100)
+# t.add('M1','D2', mean=0.5, size=100)
+# t.add('M2','D1', mean=0.2, size=100)
+# t.add('M2','D2', mean=0.1, size=100)
+# t.add('M2','D3', mean=0.7, size=100)
+# t.add('M2','D4', mean=0.3, size=100)
+# t.add('M3','D1', mean=0.9, size=100)
+# t.add('M3','D2', mean=0, size=100)
+#
+# print(t.latextabular())
+#
+# print('rank')
+# print(t.get_rank_table().latextabular())

quapy/data/base.py

@@ -186,7 +186,7 @@ class Dataset:
     def stats(self):
         tr_stats = self.training.stats(show=False)
         te_stats = self.test.stats(show=False)
-        print(f'Name={self.name} #tr-instances={tr_stats["instances"]}, #te-instances={te_stats["instances"]}, '
+        print(f'Dataset={self.name} #tr-instances={tr_stats["instances"]}, #te-instances={te_stats["instances"]}, '
               f'type={tr_stats["type"]}, #features={tr_stats["features"]}, #classes={tr_stats["classes"]}, '
               f'tr-prevs={tr_stats["prevs"]}, te-prevs={te_stats["prevs"]}')
 

quapy/method/aggregative.py

@@ -408,11 +408,12 @@ class ELM(AggregativeQuantifier, BinaryQuantifier):
         self.svmperf_base = svmperf_base
         self.loss = loss
         self.kwargs = kwargs
+        self.learner = SVMperf(self.svmperf_base, loss=self.loss, **self.kwargs)
 
     def fit(self, data: LabelledCollection, fit_learner=True):
         self._check_binary(data, self.__class__.__name__)
         assert fit_learner, 'the method requires that fit_learner=True'
-        self.learner = SVMperf(self.svmperf_base, loss=self.loss, **self.kwargs).fit(data.instances, data.labels)
+        self.learner.fit(data.instances, data.labels)
         return self
 
     def aggregate(self, classif_predictions:np.ndarray):