From 9542eaee61d17db15d80a3f2cde371112aef6076 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 22 Feb 2024 15:10:45 +0100 Subject: [PATCH] doing some benchmarking --- quapy/util.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/quapy/util.py b/quapy/util.py index dfaef57..7f0abc4 100644 --- a/quapy/util.py +++ b/quapy/util.py @@ -56,6 +56,7 @@ def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'): :param seed: the numeric seed :param asarray: set to True to return a np.ndarray instead of a list :param backend: indicates the backend used for handling parallel works + :param open_args: if True, then the delayed function is called on *args_i, instead of on args_i """ def func_dec(environ, seed, *args): qp.environ = environ.copy() @@ -74,6 +75,40 @@ def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'): return out +def parallel_unpack(func, args, n_jobs, seed=None, asarray=True, backend='loky'): + """ + A wrapper of multiprocessing: + + >>> Parallel(n_jobs=n_jobs)( + >>> delayed(func)(*args_i) for args_i in args + >>> ) + + that takes the `quapy.environ` variable as input silently. + Seeds the child processes to ensure reproducibility when n_jobs>1. + + :param func: callable + :param args: args of func + :param seed: the numeric seed + :param asarray: set to True to return a np.ndarray instead of a list + :param backend: indicates the backend used for handling parallel works + """ + + def func_dec(environ, seed, *args): + qp.environ = environ.copy() + qp.environ['N_JOBS'] = 1 + # set a context with a temporal seed to ensure results are reproducibles in parallel + with ExitStack() as stack: + if seed is not None: + stack.enter_context(qp.util.temp_seed(seed)) + return func(*args) + + out = Parallel(n_jobs=n_jobs, backend=backend)( + delayed(func_dec)(qp.environ, None if seed is None else seed + i, *args_i) for i, args_i in enumerate(args) + ) + if asarray: + out = np.asarray(out) + return out + @contextlib.contextmanager def temp_seed(random_state): """