ref no longer needed in configuration

.vscode/launch.json

@@ -11,7 +11,7 @@
             "request": "launch",
             "program": "C:\\Users\\Lorenzo Volpi\\source\\tesi\\quacc\\main.py",
             "console": "integratedTerminal",
-            "justMyCode": false
+            "justMyCode": true
         },
         {
             "name": "main_test",

conf.yaml

@@ -13,7 +13,6 @@ debug_conf: &debug_conf
     debug:
       PLOT_ESTIMATORS:
         - mul_sld
-        - ref
         - atc_mc
       PLOT_STDEV: true
 
@@ -107,4 +106,4 @@ main_conf: &main_conf
       - atc_ne
       - doc_feat
 
-exec: *test_conf
+exec: *debug_conf

quacc.log

@@ -1588,3 +1588,51 @@
 03/11/23 22:55:49| INFO        mul_sld_gs finished [took 306.4505s]
 03/11/23 23:00:15| INFO        bin_sld_gs finished [took 573.7761s]
 03/11/23 23:00:15| INFO     Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 576.7586s]
+----------------------------------------------------------------------------------------------------
+03/11/23 23:33:15| INFO     dataset imdb_1prevs
+03/11/23 23:33:22| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+03/11/23 23:33:22| ERROR    Evaluation over imdb_1prevs failed. Exception: 'function' object is not iterable
+03/11/23 23:33:22| ERROR    Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
+----------------------------------------------------------------------------------------------------
+03/11/23 23:34:15| INFO     dataset imdb_1prevs
+03/11/23 23:34:23| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+03/11/23 23:34:35| INFO        atc_mc finished [took 11.5081s]
+03/11/23 23:34:45| INFO        ref finished [took 8.7754s]
+03/11/23 23:34:47| INFO        mul_sld finished [took 22.9651s]
+03/11/23 23:34:47| INFO     Dataset sample 0.50 of dataset imdb_1prevs finished [took 23.9721s]
+03/11/23 23:34:47| ERROR    Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref'
+----------------------------------------------------------------------------------------------------
+03/11/23 23:36:10| INFO     dataset imdb_1prevs
+03/11/23 23:36:30| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+03/11/23 23:38:02| INFO        atc_mc finished [took 56.2957s]
+03/11/23 23:38:03| INFO        mul_sld finished [took 57.6237s]
+03/11/23 23:38:40| INFO        ref finished [took 37.7811s]
+03/11/23 23:38:40| INFO     Dataset sample 0.50 of dataset imdb_1prevs finished [took 130.9031s]
+03/11/23 23:38:42| ERROR    Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref'
+----------------------------------------------------------------------------------------------------
+03/11/23 23:39:32| INFO     dataset imdb_1prevs
+03/11/23 23:39:42| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+03/11/23 23:40:08| INFO        atc_mc finished [took 24.7110s]
+03/11/23 23:40:23| INFO        mul_sld finished [took 40.2345s]
+03/11/23 23:40:26| INFO        ref finished [took 17.8417s]
+03/11/23 23:40:26| INFO     Dataset sample 0.50 of dataset imdb_1prevs finished [took 44.8087s]
+----------------------------------------------------------------------------------------------------
+03/11/23 23:41:18| INFO     dataset imdb_1prevs
+03/11/23 23:41:28| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+03/11/23 23:41:54| INFO        atc_mc finished [took 24.0569s]
+03/11/23 23:42:03| INFO        mul_sld finished [took 33.3390s]
+03/11/23 23:42:12| INFO        ref finished [took 16.9551s]
+03/11/23 23:42:12| INFO     Dataset sample 0.50 of dataset imdb_1prevs finished [took 43.2484s]
+----------------------------------------------------------------------------------------------------
+04/11/23 00:03:17| ERROR    Evaluation over imdb_1prevs failed. Exception: CompEstimatorName_.__init__() missing 1 required positional argument: 'ce'
+04/11/23 00:03:17| ERROR    Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
+----------------------------------------------------------------------------------------------------
+04/11/23 00:03:50| ERROR    Evaluation over imdb_1prevs failed. Exception: 'CompEstimator' object has no attribute '_CompEstimatorName___get'
+04/11/23 00:03:50| ERROR    Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
+----------------------------------------------------------------------------------------------------
+04/11/23 00:04:42| INFO     dataset imdb_1prevs
+04/11/23 00:04:53| INFO     Dataset sample 0.50 of dataset imdb_1prevs started
+04/11/23 00:05:13| INFO        ref finished [took 19.2363s]
+04/11/23 00:05:20| INFO        atc_mc finished [took 26.4278s]
+04/11/23 00:05:29| INFO        mul_sld finished [took 35.3110s]
+04/11/23 00:05:29| INFO     Dataset sample 0.50 of dataset imdb_1prevs finished [took 36.4422s]

quacc/evaluation/comp.py

@@ -3,6 +3,7 @@ import time
 from traceback import print_exception as traceback
 from typing import List
 
+import numpy as np
 import pandas as pd
 import quapy as qp
 
@@ -17,31 +18,63 @@ pd.set_option("display.float_format", "{:.4f}".format)
 qp.environ["SAMPLE_SIZE"] = env.SAMPLE_SIZE
 
 
+class CompEstimatorName_:
+    def __init__(self, ce):
+        self.ce = ce
+
+    def __getitem__(self, e: str | List[str]):
+        if isinstance(e, str):
+            return self.ce._CompEstimator__get(e)[0]
+        elif isinstance(e, list):
+            return list(self.ce._CompEstimator__get(e).keys())
+
+
+class CompEstimatorFunc_:
+    def __init__(self, ce):
+        self.ce = ce
+
+    def __getitem__(self, e: str | List[str]):
+        if isinstance(e, str):
+            return self.ce._CompEstimator__get(e)[1]
+        elif isinstance(e, list):
+            return list(self.ce._CompEstimator__get(e).values())
+
+
 class CompEstimator:
     __dict = method._methods | baseline._baselines
 
-    def __class_getitem__(cls, e: str | List[str]):
+    def __get(cls, e: str | List[str]):
         if isinstance(e, str):
             try:
-                return cls.__dict[e]
+                return (e, cls.__dict[e])
             except KeyError:
                 raise KeyError(f"Invalid estimator: estimator {e} does not exist")
         elif isinstance(e, list):
-            _subtr = [k for k in e if k not in cls.__dict]
+            _subtr = np.setdiff1d(e, list(cls.__dict.keys()))
             if len(_subtr) > 0:
                 raise KeyError(
                     f"Invalid estimator: estimator {_subtr[0]} does not exist"
                 )
 
-            return [fun for k, fun in cls.__dict.items() if k in e]
+            e_fun = {k: fun for k, fun in cls.__dict.items() if k in e}
+            if "ref" not in e:
+                e_fun["ref"] = cls.__dict["ref"]
+
+            return e_fun
+
+    @property
+    def name(self):
+        return CompEstimatorName_(self)
+
+    @property
+    def func(self):
+        return CompEstimatorFunc_(self)
 
 
-CE = CompEstimator
+CE = CompEstimator()
 
 
-def evaluate_comparison(
+def evaluate_comparison(dataset: Dataset, estimators=None) -> EvaluationReport:
-    dataset: Dataset, estimators=["OUR_BIN_SLD", "OUR_MUL_SLD"]
-) -> EvaluationReport:
     log = Logger.logger()
     # with multiprocessing.Pool(1) as pool:
     with multiprocessing.Pool(len(estimators)) as pool:
@@ -52,7 +85,9 @@ def evaluate_comparison(
                 f"Dataset sample {d.train_prev[1]:.2f} of dataset {dataset.name} started"
             )
             tstart = time.time()
-            tasks = [(estim, d.train, d.validation, d.test) for estim in CE[estimators]]
+            tasks = [
+                (estim, d.train, d.validation, d.test) for estim in CE.func[estimators]
+            ]
             results = [
                 pool.apply_async(estimate_worker, t, {"_env": env, "q": Logger.queue()})
                 for t in tasks

quacc/main.py

@@ -7,6 +7,8 @@ from quacc.environment import env
 from quacc.logger import Logger
 from quacc.utils import create_dataser_dir
 
+CE = comp.CompEstimator()
+
 
 def toast():
     if platform == "win32":
@@ -26,7 +28,10 @@ def estimate_comparison():
         )
         create_dataser_dir(dataset.name, update=env.DATASET_DIR_UPDATE)
         try:
-            dr = comp.evaluate_comparison(dataset, estimators=env.COMP_ESTIMATORS)
+            dr = comp.evaluate_comparison(
+                dataset,
+                estimators=CE.name[env.COMP_ESTIMATORS],
+            )
         except Exception as e:
             log.error(f"Evaluation over {dataset.name} failed. Exception: {e}")
             traceback(e)
@@ -37,7 +42,7 @@ def estimate_comparison():
                     _repr = dr.to_md(
                         conf=plot_conf,
                         metric=m,
-                        estimators=env.PLOT_ESTIMATORS,
+                        estimators=CE.name[env.PLOT_ESTIMATORS],
                         stdev=env.PLOT_STDEV,
                     )
                     with open(output_path, "w") as f: