forked from moreo/QuaPy
90 lines
2.7 KiB
Python
90 lines
2.7 KiB
Python
import numpy as np
|
|
|
|
|
|
nice = {
|
|
'mae':'AE',
|
|
'mrae':'RAE',
|
|
'ae':'AE',
|
|
'rae':'RAE',
|
|
'svmkld': 'SVM(KLD)',
|
|
'svmnkld': 'SVM(NKLD)',
|
|
'svmq': 'SVM(Q)',
|
|
'svmae': 'SVM(AE)',
|
|
'svmnae': 'SVM(NAE)',
|
|
'svmmae': 'SVM(AE)',
|
|
'svmmrae': 'SVM(RAE)',
|
|
'quanet': 'QuaNet',
|
|
'hdy': 'HDy',
|
|
'dys': 'DyS',
|
|
'epaccmaeptr': 'E(PACC)$_\mathrm{Ptr}$',
|
|
'epaccmaemae': 'E(PACC)$_\mathrm{AE}$',
|
|
'epaccmraeptr': 'E(PACC)$_\mathrm{Ptr}$',
|
|
'epaccmraemrae': 'E(PACC)$_\mathrm{RAE}$',
|
|
'svmperf':'',
|
|
'sanders': 'Sanders',
|
|
'semeval13': 'SemEval13',
|
|
'semeval14': 'SemEval14',
|
|
'semeval15': 'SemEval15',
|
|
'semeval16': 'SemEval16',
|
|
'Average': 'Average'
|
|
}
|
|
|
|
|
|
def nicerm(key):
|
|
return '\mathrm{'+nice[key]+'}'
|
|
|
|
|
|
def nicename(method, eval_name=None, side=False):
|
|
m = nice.get(method, method.upper())
|
|
if eval_name is not None:
|
|
o = '$^{' + nicerm(eval_name) + '}$'
|
|
m = (m+o).replace('$$','')
|
|
if side:
|
|
m = '\side{'+m+'}'
|
|
return m
|
|
|
|
|
|
def load_Gao_Sebastiani_previous_results():
|
|
def rename(method):
|
|
old2new = {
|
|
'kld': 'svmkld',
|
|
'nkld': 'svmnkld',
|
|
'qbeta2': 'svmq',
|
|
'em': 'sld'
|
|
}
|
|
return old2new.get(method, method)
|
|
|
|
gao_seb_results = {}
|
|
with open('./Gao_Sebastiani_results.txt', 'rt') as fin:
|
|
lines = fin.readlines()
|
|
for line in lines[1:]:
|
|
line = line.strip()
|
|
parts = line.lower().split()
|
|
if len(parts) == 4:
|
|
dataset, method, ae, rae = parts
|
|
else:
|
|
method, ae, rae = parts
|
|
learner, method = method.split('-')
|
|
method = rename(method)
|
|
gao_seb_results[f'{dataset}-{method}-ae'] = float(ae)
|
|
gao_seb_results[f'{dataset}-{method}-rae'] = float(rae)
|
|
return gao_seb_results
|
|
|
|
|
|
def get_ranks_from_Gao_Sebastiani():
|
|
gao_seb_results = load_Gao_Sebastiani_previous_results()
|
|
datasets = set([key.split('-')[0] for key in gao_seb_results.keys()])
|
|
methods = np.sort(np.unique([key.split('-')[1] for key in gao_seb_results.keys()]))
|
|
ranks = {}
|
|
for metric in ['ae', 'rae']:
|
|
for dataset in datasets:
|
|
scores = [gao_seb_results[f'{dataset}-{method}-{metric}'] for method in methods]
|
|
order = np.argsort(scores)
|
|
sorted_methods = methods[order]
|
|
for i, method in enumerate(sorted_methods):
|
|
ranks[f'{dataset}-{method}-{metric}'] = i+1
|
|
for method in methods:
|
|
rankave = np.mean([ranks[f'{dataset}-{method}-{metric}'] for dataset in datasets])
|
|
ranks[f'Average-{method}-{metric}'] = rankave
|
|
return ranks, gao_seb_results
|