import pickle import os from time import time from collections import defaultdict import numpy as np from sklearn.linear_model import LogisticRegression import quapy as qp from KDEy.kdey_devel import KDEyMLauto, KDEyMLauto2 from quapy.method.aggregative import PACC, EMQ, KDEyML from quapy.model_selection import GridSearchQ from quapy.protocol import UPP from pathlib import Path from result_table.src.table import Table SEED = 1 def newLR(): return LogisticRegression(max_iter=3000) # typical hyperparameters explored for Logistic Regression logreg_grid = { 'C': np.logspace(-4,4,9), 'class_weight': [None, 'balanced'] } def wrap_hyper(classifier_hyper_grid: dict): return {'classifier__' + k: v for k, v in classifier_hyper_grid.items()} METHODS = [ # ('PACC', PACC(newLR()), wrap_hyper(logreg_grid)), # ('EMQ', EMQ(newLR()), wrap_hyper(logreg_grid)),/ ('KDEy', KDEyML(newLR()), {**wrap_hyper(logreg_grid), **{'bandwidth': np.logspace(-4, np.log10(0.2), 20)}}), # ('KDEy-MLred', KDEyMLred(newLR()), {**wrap_hyper(logreg_grid), **{'bandwidth': np.logspace(-4, np.log10(0.2), 20)}}), ('KDEy-scott', KDEyML(newLR(), bandwidth='scott'), wrap_hyper(logreg_grid)), ('KDEy-silver', KDEyML(newLR(), bandwidth='silverman'), wrap_hyper(logreg_grid)), ('KDEy-NLL', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood', search='grid'), wrap_hyper(logreg_grid)), ('KDEy-NLL+', KDEyMLauto2(newLR(), bandwidth='auto', target='likelihood', search='optim'), wrap_hyper(logreg_grid)), ('KDEy-AE', KDEyMLauto2(newLR(), bandwidth='auto', target='mae', search='grid'), wrap_hyper(logreg_grid)), ('KDEy-AE+', KDEyMLauto2(newLR(), bandwidth='auto', target='mae', search='optim'), wrap_hyper(logreg_grid)), ('KDEy-RAE', KDEyMLauto2(newLR(), bandwidth='auto', target='mrae', search='grid'), wrap_hyper(logreg_grid)), ('KDEy-RAE+', KDEyMLauto2(newLR(), bandwidth='auto', target='mrae', search='optim'), wrap_hyper(logreg_grid)), ] """ TKDEyML era primero bandwidth (init 0.05) y luego prevalence (init uniform) TKDEyML2 era primero prevalence (init uniform) y luego bandwidth (init 0.05) TKDEyML3 era primero prevalence (init uniform) y luego bandwidth (init 0.1) TKDEyML4 es como ML2 pero max 5 iteraciones por optimización """ TRANSDUCTIVE_METHODS = [ #('TKDEy-ML', KDEyMLauto(newLR()), None), # ('TKDEy-both', KDEyMLauto(newLR(), optim='both'), None), # ('TKDEy-bothfine', KDEyMLauto(newLR(), optim='both_fine'), None), # ('TKDEy-two', KDEyMLauto(newLR(), optim='two_steps'), None), # ('TKDEy-MLike', KDEyMLauto(newLR(), optim='max_likelihood'), None), # ('TKDEy-MLike2', KDEyMLauto(newLR(), optim='max_likelihood2'), None), #('TKDEy-ML3', KDEyMLauto(newLR()), None), #('TKDEy-ML4', KDEyMLauto(newLR()), None), ] def show_results(result_path, tables, tables_path='./tables/main.pdf'): import pandas as pd df = pd.read_csv(result_path + '.csv', sep='\t') pd.set_option('display.max_columns', None) pd.set_option('display.max_rows', None) pd.set_option('display.width', 1000) # Ajustar el ancho máximo pv = df.pivot_table(index='Dataset', columns="Method", values=["MAE"], margins=True) print(pv) pv = df.pivot_table(index='Dataset', columns="Method", values=["MRAE"], margins=True) print(pv) pv = df.pivot_table(index='Dataset', columns="Method", values=["KLD"], margins=True) print(pv) pv = df.pivot_table(index='Dataset', columns="Method", values=["TR-TIME"], margins=True) print(pv) pv = df.pivot_table(index='Dataset', columns="Method", values=["TE-TIME"], margins=True) print(pv) os.makedirs(Path(tables_path).parent, exist_ok=True) tables= [table for table in tables.values()] method_replace = { 'KDEy': 'KDEy(orig)', 'KDEy-scott': 'Scott', 'KDEy-silver': 'Silver', 'KDEy-NLL': 'NLL(grid)', 'KDEy-NLL+': 'NLL(search)', 'KDEy-AE': 'AE(grid)', 'KDEy-AE+': 'AE(search)', 'KDEy-RAE': 'RAE(grid)', 'KDEy-RAE+': 'RAE(search)', } Table.LatexPDF(tables_path, tables, method_replace=method_replace, verbose=True, clean=False) def collect_results(method_name, tables): print('Init method', method_name) with open(global_result_path + '.csv', 'at') as csv: for dataset in qp.datasets.UCI_MULTICLASS_DATASETS: print('init', dataset) # run_experiment(global_result_path, method_name, quantifier, param_grid, dataset) local_result_path = os.path.join(Path(global_result_path).parent, method_name + '_' + dataset + '.dataframe') if os.path.exists(local_result_path): print(f'result file {local_result_path} already exist; skipping') report = qp.util.load_report(local_result_path) for metric, table in tables.items(): add_column = metric in ['tr_time', 'te_time'] if not add_column: add_column = (metric=='mrae' and '-AE' not in method_name) or (metric=='mae' and '-RAE' not in method_name) if add_column: tables[metric].add(benchmark=dataset, method=method_name, v=report[metric]) # tables['mrae'].add(benchmark=dataset, method=method_name, v=report['mrae']) else: continue means = report.mean(numeric_only=True) csv.write(f'{method_name}\t{dataset}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\t{means["kld"]:.5f}\t{means["tr_time"]:.3f}\t{means["te_time"]:.3f}\n') csv.flush() if __name__ == '__main__': qp.environ['SAMPLE_SIZE'] = 500 qp.environ['N_JOBS'] = -1 n_bags_val = 100 n_bags_test = 500 result_dir = f'results_quantification/ucimulti' os.makedirs(result_dir, exist_ok=True) tables = { 'mae': Table('inductive-mae'), 'mrae': Table('inductive-mrae'), 'tr_time': Table('inductive-tr-time'), # 'te_time': Table('inductive-te-time'), } tables['tr_time'].format.show_std = False # tables['te_time'].format.show_std = False global_result_path = f'{result_dir}/allmethods' with open(global_result_path + '.csv', 'wt') as csv: csv.write(f'Method\tDataset\tMAE\tMRAE\tKLD\tTR-TIME\tTE-TIME\n') for method_name, _, _ in METHODS + TRANSDUCTIVE_METHODS: collect_results(method_name, tables) show_results(global_result_path, tables)