ref no longer needed in configuration
This commit is contained in:
Lorenzo Volpi
parent
493fb931a9
commit
a47de4e0cb
|
|
@ -11,7 +11,7 @@
|
|||
"request": "launch",
|
||||
"program": "C:\\Users\\Lorenzo Volpi\\source\\tesi\\quacc\\main.py",
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": false
|
||||
"justMyCode": true
|
||||
},
|
||||
{
|
||||
"name": "main_test",
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ debug_conf: &debug_conf
|
|||
debug:
|
||||
PLOT_ESTIMATORS:
|
||||
- mul_sld
|
||||
- ref
|
||||
- atc_mc
|
||||
PLOT_STDEV: true
|
||||
|
||||
|
|
@ -107,4 +106,4 @@ main_conf: &main_conf
|
|||
- atc_ne
|
||||
- doc_feat
|
||||
|
||||
exec: *test_conf
|
||||
exec: *debug_conf
|
||||
48
quacc.log
48
quacc.log
|
|
@ -1588,3 +1588,51 @@
|
|||
03/11/23 22:55:49| INFO mul_sld_gs finished [took 306.4505s]
|
||||
03/11/23 23:00:15| INFO bin_sld_gs finished [took 573.7761s]
|
||||
03/11/23 23:00:15| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 576.7586s]
|
||||
----------------------------------------------------------------------------------------------------
|
||||
03/11/23 23:33:15| INFO dataset imdb_1prevs
|
||||
03/11/23 23:33:22| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
03/11/23 23:33:22| ERROR Evaluation over imdb_1prevs failed. Exception: 'function' object is not iterable
|
||||
03/11/23 23:33:22| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
|
||||
----------------------------------------------------------------------------------------------------
|
||||
03/11/23 23:34:15| INFO dataset imdb_1prevs
|
||||
03/11/23 23:34:23| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
03/11/23 23:34:35| INFO atc_mc finished [took 11.5081s]
|
||||
03/11/23 23:34:45| INFO ref finished [took 8.7754s]
|
||||
03/11/23 23:34:47| INFO mul_sld finished [took 22.9651s]
|
||||
03/11/23 23:34:47| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 23.9721s]
|
||||
03/11/23 23:34:47| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref'
|
||||
----------------------------------------------------------------------------------------------------
|
||||
03/11/23 23:36:10| INFO dataset imdb_1prevs
|
||||
03/11/23 23:36:30| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
03/11/23 23:38:02| INFO atc_mc finished [took 56.2957s]
|
||||
03/11/23 23:38:03| INFO mul_sld finished [took 57.6237s]
|
||||
03/11/23 23:38:40| INFO ref finished [took 37.7811s]
|
||||
03/11/23 23:38:40| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 130.9031s]
|
||||
03/11/23 23:38:42| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: 'ref'
|
||||
----------------------------------------------------------------------------------------------------
|
||||
03/11/23 23:39:32| INFO dataset imdb_1prevs
|
||||
03/11/23 23:39:42| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
03/11/23 23:40:08| INFO atc_mc finished [took 24.7110s]
|
||||
03/11/23 23:40:23| INFO mul_sld finished [took 40.2345s]
|
||||
03/11/23 23:40:26| INFO ref finished [took 17.8417s]
|
||||
03/11/23 23:40:26| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 44.8087s]
|
||||
----------------------------------------------------------------------------------------------------
|
||||
03/11/23 23:41:18| INFO dataset imdb_1prevs
|
||||
03/11/23 23:41:28| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
03/11/23 23:41:54| INFO atc_mc finished [took 24.0569s]
|
||||
03/11/23 23:42:03| INFO mul_sld finished [took 33.3390s]
|
||||
03/11/23 23:42:12| INFO ref finished [took 16.9551s]
|
||||
03/11/23 23:42:12| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 43.2484s]
|
||||
----------------------------------------------------------------------------------------------------
|
||||
04/11/23 00:03:17| ERROR Evaluation over imdb_1prevs failed. Exception: CompEstimatorName_.__init__() missing 1 required positional argument: 'ce'
|
||||
04/11/23 00:03:17| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
|
||||
----------------------------------------------------------------------------------------------------
|
||||
04/11/23 00:03:50| ERROR Evaluation over imdb_1prevs failed. Exception: 'CompEstimator' object has no attribute '_CompEstimatorName___get'
|
||||
04/11/23 00:03:50| ERROR Failed while saving configuration imdb_debug of imdb_1prevs. Exception: cannot access local variable 'dr' where it is not associated with a value
|
||||
----------------------------------------------------------------------------------------------------
|
||||
04/11/23 00:04:42| INFO dataset imdb_1prevs
|
||||
04/11/23 00:04:53| INFO Dataset sample 0.50 of dataset imdb_1prevs started
|
||||
04/11/23 00:05:13| INFO ref finished [took 19.2363s]
|
||||
04/11/23 00:05:20| INFO atc_mc finished [took 26.4278s]
|
||||
04/11/23 00:05:29| INFO mul_sld finished [took 35.3110s]
|
||||
04/11/23 00:05:29| INFO Dataset sample 0.50 of dataset imdb_1prevs finished [took 36.4422s]
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import time
|
|||
from traceback import print_exception as traceback
|
||||
from typing import List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import quapy as qp
|
||||
|
||||
|
|
@ -17,31 +18,63 @@ pd.set_option("display.float_format", "{:.4f}".format)
|
|||
qp.environ["SAMPLE_SIZE"] = env.SAMPLE_SIZE
|
||||
|
||||
|
||||
class CompEstimatorName_:
|
||||
def __init__(self, ce):
|
||||
self.ce = ce
|
||||
|
||||
def __getitem__(self, e: str | List[str]):
|
||||
if isinstance(e, str):
|
||||
return self.ce._CompEstimator__get(e)[0]
|
||||
elif isinstance(e, list):
|
||||
return list(self.ce._CompEstimator__get(e).keys())
|
||||
|
||||
|
||||
class CompEstimatorFunc_:
|
||||
def __init__(self, ce):
|
||||
self.ce = ce
|
||||
|
||||
def __getitem__(self, e: str | List[str]):
|
||||
if isinstance(e, str):
|
||||
return self.ce._CompEstimator__get(e)[1]
|
||||
elif isinstance(e, list):
|
||||
return list(self.ce._CompEstimator__get(e).values())
|
||||
|
||||
|
||||
class CompEstimator:
|
||||
__dict = method._methods | baseline._baselines
|
||||
|
||||
def __class_getitem__(cls, e: str | List[str]):
|
||||
def __get(cls, e: str | List[str]):
|
||||
if isinstance(e, str):
|
||||
try:
|
||||
return cls.__dict[e]
|
||||
return (e, cls.__dict[e])
|
||||
except KeyError:
|
||||
raise KeyError(f"Invalid estimator: estimator {e} does not exist")
|
||||
elif isinstance(e, list):
|
||||
_subtr = [k for k in e if k not in cls.__dict]
|
||||
_subtr = np.setdiff1d(e, list(cls.__dict.keys()))
|
||||
if len(_subtr) > 0:
|
||||
raise KeyError(
|
||||
f"Invalid estimator: estimator {_subtr[0]} does not exist"
|
||||
)
|
||||
|
||||
return [fun for k, fun in cls.__dict.items() if k in e]
|
||||
e_fun = {k: fun for k, fun in cls.__dict.items() if k in e}
|
||||
if "ref" not in e:
|
||||
e_fun["ref"] = cls.__dict["ref"]
|
||||
|
||||
return e_fun
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return CompEstimatorName_(self)
|
||||
|
||||
@property
|
||||
def func(self):
|
||||
return CompEstimatorFunc_(self)
|
||||
|
||||
|
||||
CE = CompEstimator
|
||||
CE = CompEstimator()
|
||||
|
||||
|
||||
def evaluate_comparison(
|
||||
dataset: Dataset, estimators=["OUR_BIN_SLD", "OUR_MUL_SLD"]
|
||||
) -> EvaluationReport:
|
||||
def evaluate_comparison(dataset: Dataset, estimators=None) -> EvaluationReport:
|
||||
log = Logger.logger()
|
||||
# with multiprocessing.Pool(1) as pool:
|
||||
with multiprocessing.Pool(len(estimators)) as pool:
|
||||
|
|
@ -52,7 +85,9 @@ def evaluate_comparison(
|
|||
f"Dataset sample {d.train_prev[1]:.2f} of dataset {dataset.name} started"
|
||||
)
|
||||
tstart = time.time()
|
||||
tasks = [(estim, d.train, d.validation, d.test) for estim in CE[estimators]]
|
||||
tasks = [
|
||||
(estim, d.train, d.validation, d.test) for estim in CE.func[estimators]
|
||||
]
|
||||
results = [
|
||||
pool.apply_async(estimate_worker, t, {"_env": env, "q": Logger.queue()})
|
||||
for t in tasks
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ from quacc.environment import env
|
|||
from quacc.logger import Logger
|
||||
from quacc.utils import create_dataser_dir
|
||||
|
||||
CE = comp.CompEstimator()
|
||||
|
||||
|
||||
def toast():
|
||||
if platform == "win32":
|
||||
|
|
@ -26,7 +28,10 @@ def estimate_comparison():
|
|||
)
|
||||
create_dataser_dir(dataset.name, update=env.DATASET_DIR_UPDATE)
|
||||
try:
|
||||
dr = comp.evaluate_comparison(dataset, estimators=env.COMP_ESTIMATORS)
|
||||
dr = comp.evaluate_comparison(
|
||||
dataset,
|
||||
estimators=CE.name[env.COMP_ESTIMATORS],
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(f"Evaluation over {dataset.name} failed. Exception: {e}")
|
||||
traceback(e)
|
||||
|
|
@ -37,7 +42,7 @@ def estimate_comparison():
|
|||
_repr = dr.to_md(
|
||||
conf=plot_conf,
|
||||
metric=m,
|
||||
estimators=env.PLOT_ESTIMATORS,
|
||||
estimators=CE.name[env.PLOT_ESTIMATORS],
|
||||
stdev=env.PLOT_STDEV,
|
||||
)
|
||||
with open(output_path, "w") as f:
|
||||
|
|
|
|||
Loading…
Reference in New Issue