lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

parallelization changed to joblib

This commit is contained in:
Lorenzo Volpi 2023-12-02 02:08:16 +01:00
parent 78c210b15f
commit 8705f2b3c0
1 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
quacc/evaluation/comp.py
View File

@ -5,6 +5,7 @@ from traceback import print_exception as traceback
import pandas as pd
import quapy as qp
from joblib import Parallel, delayed
from quacc.dataset import Dataset
from quacc.environment import env
@ -21,7 +22,7 @@ def evaluate_comparison(dataset: Dataset, estimators=None) -> DatasetReport:
log = Logger.logger()
# with multiprocessing.Pool(1) as pool:
__pool_size = round(os.cpu_count() * 0.8)
with multiprocessing.Pool(__pool_size) as pool:
# with multiprocessing.Pool(__pool_size) as pool:
dr = DatasetReport(dataset.name)
log.info(f"dataset {dataset.name} [pool size: {__pool_size}]")
for d in dataset():
@ -41,9 +42,13 @@ def evaluate_comparison(dataset: Dataset, estimators=None) -> DatasetReport:
]
try:
tstart = time.time()
results = [
r for r in pool.imap(estimate_worker, tasks) if r is not None
]
results = Parallel(n_jobs=1)(delayed(estimate_worker)(t) for t in tasks)
results = [r for r in results if r is not None]
# # r for r in pool.imap(estimate_worker, tasks) if r is not None
# r
# for r in map(estimate_worker, tasks)
# if r is not None
# ]
g_time = time.time() - tstart
log.info(