diff --git a/.vscode/launch.json b/.vscode/launch.json index 429433a..5145d34 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "request": "launch", "program": "C:\\Users\\Lorenzo Volpi\\source\\tesi\\quacc\\main.py", "console": "integratedTerminal", - "justMyCode": false + "justMyCode": true }, { "name": "main_test", diff --git a/conf.yaml b/conf.yaml index a75a718..6ec7356 100644 --- a/conf.yaml +++ b/conf.yaml @@ -13,7 +13,6 @@ debug_conf: &debug_conf debug: PLOT_ESTIMATORS: - mul_sld - - ref - atc_mc PLOT_STDEV: true @@ -107,4 +106,4 @@ main_conf: &main_conf - atc_ne - doc_feat -exec: *test_conf \ No newline at end of file +exec: *debug_conf \ No newline at end of file diff --git a/quacc.log b/quacc.log index df47722..e229551 100644 --- a/quacc.log +++ b/quacc.log @@ -1588,3 +1588,51 @@ 03/11/23 22:55:49| INFO mul_sld_gs finished [took 306.4505s] 03/11/23 23:00:15| INFO bin_sld_gs finished [took 573.7761s] 03/11/23 23:00:15| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 576.7586s] +---------------------------------------------------------------------------------------------------- +03/11/23 23:33:15| INFO dataset imdb_1prevs +03/11/23 23:33:22| INFO Dataset sample 0.50 of dataset imdb_1prevs started +03/11/23 23:33:22| ERROR Evaluation over imdb_1prevs failed. Exception: 'function' object is not iterable +03/11/23 23:33:22| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value +---------------------------------------------------------------------------------------------------- +03/11/23 23:34:15| INFO dataset imdb_1prevs +03/11/23 23:34:23| INFO Dataset sample 0.50 of dataset imdb_1prevs started +03/11/23 23:34:35| INFO atc_mc finished [took 11.5081s] +03/11/23 23:34:45| INFO ref finished [took 8.7754s] +03/11/23 23:34:47| INFO mul_sld finished [took 22.9651s] +03/11/23 23:34:47| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 23.9721s] +03/11/23 23:34:47| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref' +---------------------------------------------------------------------------------------------------- +03/11/23 23:36:10| INFO dataset imdb_1prevs +03/11/23 23:36:30| INFO Dataset sample 0.50 of dataset imdb_1prevs started +03/11/23 23:38:02| INFO atc_mc finished [took 56.2957s] +03/11/23 23:38:03| INFO mul_sld finished [took 57.6237s] +03/11/23 23:38:40| INFO ref finished [took 37.7811s] +03/11/23 23:38:40| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 130.9031s] +03/11/23 23:38:42| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref' +---------------------------------------------------------------------------------------------------- +03/11/23 23:39:32| INFO dataset imdb_1prevs +03/11/23 23:39:42| INFO Dataset sample 0.50 of dataset imdb_1prevs started +03/11/23 23:40:08| INFO atc_mc finished [took 24.7110s] +03/11/23 23:40:23| INFO mul_sld finished [took 40.2345s] +03/11/23 23:40:26| INFO ref finished [took 17.8417s] +03/11/23 23:40:26| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 44.8087s] +---------------------------------------------------------------------------------------------------- +03/11/23 23:41:18| INFO dataset imdb_1prevs +03/11/23 23:41:28| INFO Dataset sample 0.50 of dataset imdb_1prevs started +03/11/23 23:41:54| INFO atc_mc finished [took 24.0569s] +03/11/23 23:42:03| INFO mul_sld finished [took 33.3390s] +03/11/23 23:42:12| INFO ref finished [took 16.9551s] +03/11/23 23:42:12| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 43.2484s] +---------------------------------------------------------------------------------------------------- +04/11/23 00:03:17| ERROR Evaluation over imdb_1prevs failed. Exception: CompEstimatorName_.__init__() missing 1 required positional argument: 'ce' +04/11/23 00:03:17| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value +---------------------------------------------------------------------------------------------------- +04/11/23 00:03:50| ERROR Evaluation over imdb_1prevs failed. Exception: 'CompEstimator' object has no attribute '_CompEstimatorName___get' +04/11/23 00:03:50| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value +---------------------------------------------------------------------------------------------------- +04/11/23 00:04:42| INFO dataset imdb_1prevs +04/11/23 00:04:53| INFO Dataset sample 0.50 of dataset imdb_1prevs started +04/11/23 00:05:13| INFO ref finished [took 19.2363s] +04/11/23 00:05:20| INFO atc_mc finished [took 26.4278s] +04/11/23 00:05:29| INFO mul_sld finished [took 35.3110s] +04/11/23 00:05:29| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 36.4422s] diff --git a/quacc/evaluation/comp.py b/quacc/evaluation/comp.py index 2f45343..6df27a6 100644 --- a/quacc/evaluation/comp.py +++ b/quacc/evaluation/comp.py @@ -3,6 +3,7 @@ import time from traceback import print_exception as traceback from typing import List +import numpy as np import pandas as pd import quapy as qp @@ -17,31 +18,63 @@ pd.set_option("display.float_format", "{:.4f}".format) qp.environ["SAMPLE_SIZE"] = env.SAMPLE_SIZE +class CompEstimatorName_: + def __init__(self, ce): + self.ce = ce + + def __getitem__(self, e: str | List[str]): + if isinstance(e, str): + return self.ce._CompEstimator__get(e)[0] + elif isinstance(e, list): + return list(self.ce._CompEstimator__get(e).keys()) + + +class CompEstimatorFunc_: + def __init__(self, ce): + self.ce = ce + + def __getitem__(self, e: str | List[str]): + if isinstance(e, str): + return self.ce._CompEstimator__get(e)[1] + elif isinstance(e, list): + return list(self.ce._CompEstimator__get(e).values()) + + class CompEstimator: __dict = method._methods | baseline._baselines - def __class_getitem__(cls, e: str | List[str]): + def __get(cls, e: str | List[str]): if isinstance(e, str): try: - return cls.__dict[e] + return (e, cls.__dict[e]) except KeyError: raise KeyError(f"Invalid estimator: estimator {e} does not exist") elif isinstance(e, list): - _subtr = [k for k in e if k not in cls.__dict] + _subtr = np.setdiff1d(e, list(cls.__dict.keys())) if len(_subtr) > 0: raise KeyError( f"Invalid estimator: estimator {_subtr[0]} does not exist" ) - return [fun for k, fun in cls.__dict.items() if k in e] + e_fun = {k: fun for k, fun in cls.__dict.items() if k in e} + if "ref" not in e: + e_fun["ref"] = cls.__dict["ref"] + + return e_fun + + @property + def name(self): + return CompEstimatorName_(self) + + @property + def func(self): + return CompEstimatorFunc_(self) -CE = CompEstimator +CE = CompEstimator() -def evaluate_comparison( - dataset: Dataset, estimators=["OUR_BIN_SLD", "OUR_MUL_SLD"] -) -> EvaluationReport: +def evaluate_comparison(dataset: Dataset, estimators=None) -> EvaluationReport: log = Logger.logger() # with multiprocessing.Pool(1) as pool: with multiprocessing.Pool(len(estimators)) as pool: @@ -52,7 +85,9 @@ def evaluate_comparison( f"Dataset sample {d.train_prev[1]:.2f} of dataset {dataset.name} started" ) tstart = time.time() - tasks = [(estim, d.train, d.validation, d.test) for estim in CE[estimators]] + tasks = [ + (estim, d.train, d.validation, d.test) for estim in CE.func[estimators] + ] results = [ pool.apply_async(estimate_worker, t, {"_env": env, "q": Logger.queue()}) for t in tasks diff --git a/quacc/main.py b/quacc/main.py index a13a65c..1fe0279 100644 --- a/quacc/main.py +++ b/quacc/main.py @@ -7,6 +7,8 @@ from quacc.environment import env from quacc.logger import Logger from quacc.utils import create_dataser_dir +CE = comp.CompEstimator() + def toast(): if platform == "win32": @@ -26,7 +28,10 @@ def estimate_comparison(): ) create_dataser_dir(dataset.name, update=env.DATASET_DIR_UPDATE) try: - dr = comp.evaluate_comparison(dataset, estimators=env.COMP_ESTIMATORS) + dr = comp.evaluate_comparison( + dataset, + estimators=CE.name[env.COMP_ESTIMATORS], + ) except Exception as e: log.error(f"Evaluation over {dataset.name} failed. Exception: {e}") traceback(e) @@ -37,7 +42,7 @@ def estimate_comparison(): _repr = dr.to_md( conf=plot_conf, metric=m, - estimators=env.PLOT_ESTIMATORS, + estimators=CE.name[env.PLOT_ESTIMATORS], stdev=env.PLOT_STDEV, ) with open(output_path, "w") as f: