diff --git a/LeQua2022/baselines_T1Amodsel.py b/LeQua2022/baselines_T1Amodsel.py index 01b02a4..d6158dc 100644 --- a/LeQua2022/baselines_T1Amodsel.py +++ b/LeQua2022/baselines_T1Amodsel.py @@ -1,12 +1,5 @@ import pickle - -import numpy as np from sklearn.linear_model import LogisticRegression -from tqdm import tqdm -import pandas as pd - -import quapy as qp -from quapy.data import LabelledCollection from quapy.method.aggregative import * import quapy.functional as F from data import * @@ -50,9 +43,11 @@ def gen_samples(): return gen_load_samples_T1(T1A_devvectors_path, nF, ground_truth_path=T1A_devprevalence_path, return_id=False) -for quantifier in [CC]: #, ACC, PCC, PACC, EMQ, HDy]: - #classifier = CalibratedClassifierCV(LogisticRegression(), n_jobs=-1) - classifier = LogisticRegression() +for quantifier in [EMQ]: # [CC, ACC, PCC, PACC, EMQ, HDy]: + if quantifier == EMQ: + classifier = CalibratedClassifierCV(LogisticRegression(), n_jobs=-1) + else: + classifier = LogisticRegression() model = quantifier(classifier) print(f'{model.__class__.__name__}: Model selection') model = qp.model_selection.GridSearchQ( diff --git a/LeQua2022/evaluation.py b/LeQua2022/evaluate.py similarity index 92% rename from LeQua2022/evaluation.py rename to LeQua2022/evaluate.py index b52c1cb..6095fd9 100644 --- a/LeQua2022/evaluation.py +++ b/LeQua2022/evaluate.py @@ -2,7 +2,6 @@ import argparse import quapy as qp from data import ResultSubmission, evaluate_submission import constants -import os """ LeQua2022 Official evaluation script @@ -20,9 +19,7 @@ def main(args): print(f'MRAE: {mrae:.4f}') if args.output is not None: - outdir = os.path.dirname(args.output) - if outdir: - os.makedirs(outdir, exist_ok=True) + qp.util.create_parent_dir(args.output) with open(args.output, 'wt') as foo: foo.write(f'MAE: {mae:.4f}\n') foo.write(f'MRAE: {mrae:.4f}\n') diff --git a/LeQua2022/predict.py b/LeQua2022/predict.py index d3e1a9b..78a9216 100644 --- a/LeQua2022/predict.py +++ b/LeQua2022/predict.py @@ -1,11 +1,11 @@ import argparse import quapy as qp -from data import ResultSubmission, evaluate_submission +from data import ResultSubmission import constants import os import pickle from tqdm import tqdm -from data import gen_load_samples_T1, load_category_map +from data import gen_load_samples_T1 from glob import glob import constants @@ -22,21 +22,16 @@ def main(args): f'dev samples ({constants.DEV_SAMPLES}) nor with the expected number of ' f'test samples ({constants.TEST_SAMPLES}).') - # _, categories = load_category_map(args.catmap) - # load pickled model model = pickle.load(open(args.model, 'rb')) # predictions predictions = ResultSubmission() - for sampleid, sample in tqdm(gen_load_samples_T1(args.samples, args.nf), - desc='predicting', total=nsamples): + for sampleid, sample in tqdm(gen_load_samples_T1(args.samples, args.nf), desc='predicting', total=nsamples): predictions.add(sampleid, model.quantify(sample)) # saving - basedir = os.path.basename(args.output) - if basedir: - os.makedirs(basedir, exist_ok=True) + qp.util.create_parent_dir(args.output) predictions.dump(args.output) diff --git a/quapy/model_selection.py b/quapy/model_selection.py index 602c5e6..35f87b9 100644 --- a/quapy/model_selection.py +++ b/quapy/model_selection.py @@ -11,6 +11,46 @@ import inspect class GridSearchQ(BaseQuantifier): + """Grid Search optimization targeting a quantification-oriented metric. + + Optimizes the hyperparameters of a quantification method, based on an evaluation method and on an evaluation + protocol for quantification. + + :param model: the quantifier to optimize + :type model: BaseQuantifier + :param param_grid: a dictionary with keys the parameter names and values the list of values to explore + :param sample_size: the size of the samples to extract from the validation set (ignored if protocl='gen') + :param protocol: either 'app' for the artificial prevalence protocol, 'npp' for the natural prevalence + protocol, or 'gen' for using a custom sampling generator function + :param n_prevpoints: if specified, indicates the number of equally distant points to extract from the interval + [0,1] in order to define the prevalences of the samples; e.g., if n_prevpoints=5, then the prevalences for + each class will be explored in [0.00, 0.25, 0.50, 0.75, 1.00]. If not specified, then eval_budget is requested. + Ignored if protocol!='app'. + :param n_repetitions: the number of repetitions for each combination of prevalences. This parameter is ignored + for the protocol='app' if eval_budget is set and is lower than the number of combinations that would be + generated using the value assigned to n_prevpoints (for the current number of classes and n_repetitions). + Ignored for protocol='npp' and protocol='gen' (use eval_budget for setting a maximum number of samples in + those cases). + :param eval_budget: if specified, sets a ceil on the number of evaluations to perform for each hyper-parameter + combination. For example, if protocol='app', there are 3 classes, n_repetitions=1 and eval_budget=20, then + n_prevpoints will be set to 5, since this will generate 15 different prevalences, i.e., [0, 0, 1], + [0, 0.25, 0.75], [0, 0.5, 0.5] ... [1, 0, 0], and since setting it to 6 would generate more than + 20. When protocol='gen', indicates the maximum number of samples to generate, but less samples will be + generated if the generator yields less samples. + :param error: an error function (callable) or a string indicating the name of an error function (valid ones + are those in qp.error.QUANTIFICATION_ERROR + :param refit: whether or not to refit the model on the whole labelled collection (training+validation) with + the best chosen hyperparameter combination. Ignored if protocol='gen' + :param val_split: either a LabelledCollection on which to test the performance of the different settings, or + a float in [0,1] indicating the proportion of labelled data to extract from the training set, or a callable + returning a generator function each time it is invoked (only for protocol='gen'). + :param n_jobs: number of parallel jobs + :param random_seed: set the seed of the random generator to replicate experiments. Ignored if protocol='gen'. + :param timeout: establishes a timer (in seconds) for each of the hyperparameters configurations being tested. + Whenever a run takes longer than this timer, that configuration will be ignored. If all configurations end up + being ignored, a TimeoutError exception is raised. If -1 (default) then no time bound is set. + :param verbose: set to True to get information through the stdout + """ def __init__(self, model: BaseQuantifier, @@ -27,43 +67,7 @@ class GridSearchQ(BaseQuantifier): random_seed=42, timeout=-1, verbose=False): - """ - Optimizes the hyperparameters of a quantification method, based on an evaluation method and on an evaluation - protocol for quantification. - :param model: the quantifier to optimize - :param param_grid: a dictionary with keys the parameter names and values the list of values to explore for - :param sample_size: the size of the samples to extract from the validation set (ignored if protocl='gen') - :param protocol: either 'app' for the artificial prevalence protocol, 'npp' for the natural prevalence - protocol, or 'gen' for using a custom sampling generator function - :param n_prevpoints: if specified, indicates the number of equally distant points to extract from the interval - [0,1] in order to define the prevalences of the samples; e.g., if n_prevpoints=5, then the prevalences for - each class will be explored in [0.00, 0.25, 0.50, 0.75, 1.00]. If not specified, then eval_budget is requested. - Ignored if protocol!='app'. - :param n_repetitions: the number of repetitions for each combination of prevalences. This parameter is ignored - for the protocol='app' if eval_budget is set and is lower than the number of combinations that would be - generated using the value assigned to n_prevpoints (for the current number of classes and n_repetitions). - Ignored for protocol='npp' and protocol='gen' (use eval_budget for setting a maximum number of samples in - those cases). - :param eval_budget: if specified, sets a ceil on the number of evaluations to perform for each hyper-parameter - combination. For example, if protocol='app', there are 3 classes, n_repetitions=1 and eval_budget=20, then - n_prevpoints will be set to 5, since this will generate 15 different prevalences, i.e., [0, 0, 1], - [0, 0.25, 0.75], [0, 0.5, 0.5] ... [1, 0, 0], and since setting it to 6 would generate more than - 20. When protocol='gen', indicates the maximum number of samples to generate, but less samples will be - generated if the generator yields less samples. - :param error: an error function (callable) or a string indicating the name of an error function (valid ones - are those in qp.error.QUANTIFICATION_ERROR - :param refit: whether or not to refit the model on the whole labelled collection (training+validation) with - the best chosen hyperparameter combination. Ignored if protocol='gen' - :param val_split: either a LabelledCollection on which to test the performance of the different settings, or - a float in [0,1] indicating the proportion of labelled data to extract from the training set, or a callable - returning a generator function each time it is invoked (only for protocol='gen'). - :param n_jobs: number of parallel jobs - :param random_seed: set the seed of the random generator to replicate experiments. Ignored if protocol='gen'. - :param timeout: establishes a timer (in seconds) for each of the hyperparameters configurations being tested. - Whenever a run takes longer than this timer, that configuration will be ignored. If all configurations end up - being ignored, a TimeoutError exception is raised. If -1 (default) then no time bound is set. - :param verbose: set to True to get information through the stdout - """ + self.model = model self.param_grid = param_grid self.sample_size = sample_size @@ -90,7 +94,7 @@ class GridSearchQ(BaseQuantifier): if self.n_prevpoints != 1: print('[warning] n_prevpoints has been set and will be ignored for the selected protocol') - def sout(self, msg): + def _sout(self, msg): if self.verbose: print(f'[{self.__class__.__name__}]: {msg}') @@ -145,10 +149,11 @@ class GridSearchQ(BaseQuantifier): raise ValueError('unknown protocol') def fit(self, training: LabelledCollection, val_split: Union[LabelledCollection, float, Callable] = None): - """ + """ Learning routine. Fits methods with all combinations of hyperparameters and selects the one minimizing + the error metric. :param training: the training set on which to optimize the hyperparameters :param val_split: either a LabelledCollection on which to test the performance of the different settings, or - a float in [0,1] indicating the proportion of labelled data to extract from the training set + a float in [0,1] indicating the proportion of labelled data to extract from the training set """ if val_split is None: val_split = self.val_split @@ -164,12 +169,12 @@ class GridSearchQ(BaseQuantifier): if self.timeout > 0: def handler(signum, frame): - self.sout('timeout reached') + self._sout('timeout reached') raise TimeoutError() signal.signal(signal.SIGALRM, handler) - self.sout(f'starting optimization with n_jobs={n_jobs}') + self._sout(f'starting optimization with n_jobs={n_jobs}') self.param_scores_ = {} self.best_score_ = None some_timeouts = False @@ -185,7 +190,7 @@ class GridSearchQ(BaseQuantifier): model.fit(training) true_prevalences, estim_prevalences = self.__generate_predictions(model, val_split) score = self.error(true_prevalences, estim_prevalences) - self.sout(f'checking hyperparams={params} got {self.error.__name__} score {score:.5f}') + self._sout(f'checking hyperparams={params} got {self.error.__name__} score {score:.5f}') if self.best_score_ is None or score < self.best_score_: self.best_score_ = score self.best_params_ = params @@ -201,15 +206,19 @@ class GridSearchQ(BaseQuantifier): if self.best_score_ is None and some_timeouts: raise TimeoutError('all jobs took more than the timeout time to end') - self.sout(f'optimization finished: best params {self.best_params_} (score={self.best_score_:.5f})') + self._sout(f'optimization finished: best params {self.best_params_} (score={self.best_score_:.5f})') if self.refit: - self.sout(f'refitting on the whole development set') + self._sout(f'refitting on the whole development set') self.best_model_.fit(training + val_split) return self def quantify(self, instances): + """Estimate class prevalence values + + :param instances: sample contanining the instances + """ assert hasattr(self, 'best_model_'), 'quantify called before fit' return self.best_model().quantify(instances) @@ -218,9 +227,18 @@ class GridSearchQ(BaseQuantifier): return self.best_model().classes_ def set_params(self, **parameters): + """Sets the hyper-parameters to explore. + + :param parameters: a dictionary with keys the parameter names and values the list of values to explore + """ self.param_grid = parameters def get_params(self, deep=True): + """Returns the dictionary of hyper-parameters to explore (`param_grid`) + + :param deep: Unused + :return: the dictionary `param_grid` + """ return self.param_grid def best_model(self): diff --git a/quapy/util.py b/quapy/util.py index ab205aa..96c5835 100644 --- a/quapy/util.py +++ b/quapy/util.py @@ -11,13 +11,12 @@ import numpy as np from joblib import Parallel, delayed -def get_parallel_slices(n_tasks, n_jobs=-1): +def _get_parallel_slices(n_tasks, n_jobs=-1): if n_jobs == -1: n_jobs = multiprocessing.cpu_count() batch = int(n_tasks / n_jobs) remainder = n_tasks % n_jobs - return [slice(job * batch, (job + 1) * batch + (remainder if job == n_jobs - 1 else 0)) for job in - range(n_jobs)] + return [slice(job * batch, (job + 1) * batch + (remainder if job == n_jobs - 1 else 0)) for job in range(n_jobs)] def map_parallel(func, args, n_jobs): @@ -26,7 +25,7 @@ def map_parallel(func, args, n_jobs): func is applied in two parallel processes to args[0:50] and to args[50:99] """ args = np.asarray(args) - slices = get_parallel_slices(len(args), n_jobs) + slices = _get_parallel_slices(len(args), n_jobs) results = Parallel(n_jobs=n_jobs)( delayed(func)(args[slice_i]) for slice_i in slices ) @@ -37,7 +36,7 @@ def parallel(func, args, n_jobs): """ A wrapper of multiprocessing: Parallel(n_jobs=n_jobs)( - delayed(func)(args_i) for args_i in args + delayed(func)(args_i) for args_i in args ) that takes the quapy.environ variable as input silently """ @@ -49,9 +48,14 @@ def parallel(func, args, n_jobs): ) - @contextlib.contextmanager def temp_seed(seed): + """ + Can be used in a "with" context to set a temporal seed without modifying the outer numpy's current state. E.g.: + with temp_seed(random_seed): + # do any computation depending on np.random functionality + :param seed: the seed to set within the "with" context + """ state = np.random.get_state() np.random.seed(seed) try: @@ -88,10 +92,30 @@ def get_quapy_home(): def create_parent_dir(path): - os.makedirs(Path(path).parent, exist_ok=True) + parentdir = Path(path).parent + if parentdir: + os.makedirs(parentdir, exist_ok=True) + + +def save_text_file(path, text): + create_parent_dir(path) + with open(text, 'wt') as fout: + fout.write(text) def pickled_resource(pickle_path:str, generation_func:callable, *args): + """ + Allows for fast reuse of resources that are generated only once by calling generation_func(*args). The next times + this function is invoked, it loads the pickled resource. Example: + def some_array(n): + return np.random.rand(n) + pickled_resource('./my_array.pkl', some_array, 10) # the resource does not exist: it is created by some_array(10) + pickled_resource('./my_array.pkl', some_array, 10) # the resource exists: it is loaded from './my_array.pkl' + :param pickle_path: the path where to save (first time) and load (next times) the resource + :param generation_func: the function that generates the resource, in case it does not exist in pickle_path + :param args: any arg that generation_func uses for generating the resources + :return: the resource + """ if pickle_path is None: return generation_func(*args) else: