From 010676df1289518e45aa4cc4cf12f32939f66db0 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Tue, 7 Oct 2025 10:27:59 +0200 Subject: [PATCH 01/43] starting devel 0.2.1 --- CHANGE_LOG.txt | 7 +++++-- quapy/__init__.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index a701186..b114e46 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -1,7 +1,10 @@ -Change Log 0.2.0 +Change Log 0.2.1 ----------------- -CLEAN TODO-FILE +... + +Change Log 0.2.0 +----------------- - Base code Refactor: - Removing coupling between LabelledCollection and quantification methods; the fit interface changes: diff --git a/quapy/__init__.py b/quapy/__init__.py index d013f5b..a952fbc 100644 --- a/quapy/__init__.py +++ b/quapy/__init__.py @@ -13,7 +13,7 @@ from . import model_selection from . import classification import os -__version__ = '0.2.0' +__version__ = '0.2.1' def _default_cls(): From d597820a59284d9001e9ddf104a4f363d94a1ec2 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 9 Oct 2025 12:10:26 +0200 Subject: [PATCH 02/43] missing doc of confidence methods --- docs/source/quapy.method.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/quapy.method.rst b/docs/source/quapy.method.rst index ac0dfc8..88fcc7d 100644 --- a/docs/source/quapy.method.rst +++ b/docs/source/quapy.method.rst @@ -60,6 +60,14 @@ quapy.method.composable module :undoc-members: :show-inheritance: +quapy.method.confidence module +------------------------------ + +.. automodule:: quapy.method.confidence + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- From 1fb8500e8765754f69381660749ce06b9b640b92 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 9 Oct 2025 12:49:08 +0200 Subject: [PATCH 03/43] improved doc --- CHANGE_LOG.txt | 2 +- docs/source/manuals/methods.md | 5 ++++- quapy/method/confidence.py | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index b114e46..3e5155f 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -1,7 +1,7 @@ Change Log 0.2.1 ----------------- -... +- Improved documentation of confidence regions. Change Log 0.2.0 ----------------- diff --git a/docs/source/manuals/methods.md b/docs/source/manuals/methods.md index 47b7cad..93aa1dd 100644 --- a/docs/source/manuals/methods.md +++ b/docs/source/manuals/methods.md @@ -604,7 +604,10 @@ estim_prevalence = model.predict(dataset.test.X) _(New in v0.2.0!)_ Some quantification methods go beyond providing a single point estimate of class prevalence values and also produce confidence regions, which characterize the uncertainty around the point estimate. In QuaPy, two such methods are currently implemented: -* Aggregative Bootstrap: The Aggregative Bootstrap method extends any aggregative quantifier by generating confidence regions for class prevalence estimates through bootstrapping. Key features of this method include: +* Aggregative Bootstrap: The Aggregative Bootstrap method extends any aggregative quantifier by generating confidence regions for class prevalence estimates through bootstrapping. The method is described in the paper [Moreo, A., Salvati, N. + An Efficient Method for Deriving Confidence Intervals in Aggregative Quantification. + Learning to Quantify: Methods and Applications (LQ 2025), co-located at ECML-PKDD 2025. + pp 12-33, Porto (Portugal)](https://lq-2025.github.io/proceedings/CompleteVolume.pdf). Key features of this method include: * Optimized Computation: The bootstrap is applied to pre-classified instances, significantly speeding up training and inference. During training, bootstrap repetitions are performed only after training the classifier once. These repetitions are used to train multiple aggregation functions. diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index f54768c..7f70bb8 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -339,6 +339,12 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): During inference, the bootstrap repetitions are applied to the pre-classified test instances. + See + `Moreo, A., Salvati, N. + An Efficient Method for Deriving Confidence Intervals in Aggregative Quantification. + Learning to Quantify: Methods and Applications (LQ 2025), co-located at ECML-PKDD 2025. + pp 12-33 `_ + :param quantifier: an aggregative quantifier :para n_train_samples: int, the number of training resamplings (defaults to 1, set to > 1 to activate a model-based bootstrap approach) @@ -437,7 +443,7 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): """ - `Bayesian quantification `_ method, + `Bayesian quantification `_ method (by Albert Ziegler and Paweł Czyż), which is a variant of :class:`ACC` that calculates the posterior probability distribution over the prevalence vectors, rather than providing a point estimate obtained by matrix inversion. From eafe486893780fc3ecb3e2c35b71fb4d61fe0599 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Mon, 20 Oct 2025 18:13:34 +0200 Subject: [PATCH 04/43] adding readme to non-aggregative --- examples/16.confidence_regions.py | 2 +- examples/18.ReadMe_for_text_analysis.py | 23 ++++ quapy/data/preprocessing.py | 43 +++++--- quapy/method/confidence.py | 26 ++++- quapy/method/non_aggregative.py | 135 +++++++++++++++--------- 5 files changed, 163 insertions(+), 66 deletions(-) create mode 100644 examples/18.ReadMe_for_text_analysis.py diff --git a/examples/16.confidence_regions.py b/examples/16.confidence_regions.py index 27fbfbd..c8e95dd 100644 --- a/examples/16.confidence_regions.py +++ b/examples/16.confidence_regions.py @@ -36,7 +36,7 @@ with qp.util.temp_seed(0): true_prev = shifted_test.prevalence() # by calling "quantify_conf", we obtain the point estimate and the confidence intervals around it - pred_prev, conf_intervals = pacc.quantify_conf(shifted_test.X) + pred_prev, conf_intervals = pacc.predict_conf(shifted_test.X) # conf_intervals is an instance of ConfidenceRegionABC, which provides some useful utilities like: # - coverage: a function which computes the fraction of true values that belong to the confidence region diff --git a/examples/18.ReadMe_for_text_analysis.py b/examples/18.ReadMe_for_text_analysis.py new file mode 100644 index 0000000..d3e1c49 --- /dev/null +++ b/examples/18.ReadMe_for_text_analysis.py @@ -0,0 +1,23 @@ +from sklearn.feature_extraction.text import CountVectorizer +import quapy as qp +from quapy.method.non_aggregative import ReadMe +import quapy.functional as F + +reviews = qp.datasets.fetch_reviews('imdb').reduce(n_train=1000, random_state=0) + +encode_0_1 = CountVectorizer(min_df=5, binary=True) +train, test = qp.data.preprocessing.instance_transformation(reviews, encode_0_1, inplace=True).train_test + +readme = ReadMe(bootstrap_trials=100, bagging_trials=100, bagging_range=100, random_state=0, verbose=True) +readme.fit(*train.Xy) + +for test_prev in [[0.25, 0.75], [0.5, 0.5], [0.75, 0.25]]: + sample = reviews.test.sampling(500, *test_prev, random_state=0) + prev_estim, conf = readme.predict_conf(sample.X) + err = qp.error.mae(sample.prevalence(), prev_estim) + print(f'true-prevalence={F.strprev(sample.prevalence())},\n' + f'predicted-prevalence={F.strprev(prev_estim)},\n' + f'MAE={err:.4f}') + print(conf) + + diff --git a/quapy/data/preprocessing.py b/quapy/data/preprocessing.py index a4be4fd..bc57d65 100644 --- a/quapy/data/preprocessing.py +++ b/quapy/data/preprocessing.py @@ -10,6 +10,36 @@ from quapy.util import map_parallel from .base import LabelledCollection +def instance_transformation(dataset:Dataset, transformer, inplace=False): + """ + Transforms a :class:`quapy.data.base.Dataset` applying the `fit_transform` and `transform` functions + of a (sklearn's) transformer. + + :param dataset: a :class:`quapy.data.base.Dataset` where the instances of training and test collections are + lists of str + :param transformer: TransformerMixin implementing `fit_transform` and `transform` functions + :param inplace: whether or not to apply the transformation inplace (True), or to a new copy (False, default) + :return: a new :class:`quapy.data.base.Dataset` with transformed instances (if inplace=False) or a reference to the + current Dataset (if inplace=True) where the instances have been transformed + """ + training_transformed = transformer.fit_transform(dataset.training.instances) + test_transformed = transformer.transform(dataset.test.instances) + + if inplace: + dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_) + dataset.test = LabelledCollection(test_transformed, dataset.test.labels, dataset.classes_) + if hasattr(transformer, 'vocabulary_'): + dataset.vocabulary = transformer.vocabulary_ + return dataset + else: + training = LabelledCollection(training_transformed, dataset.training.labels.copy(), dataset.classes_) + test = LabelledCollection(test_transformed, dataset.test.labels.copy(), dataset.classes_) + if hasattr(transformer, 'vocabulary_'): + return Dataset(training, test, transformer.vocabulary_) + else: + return Dataset(training, test) + + def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs): """ Transforms a :class:`quapy.data.base.Dataset` of textual instances into a :class:`quapy.data.base.Dataset` of @@ -29,18 +59,7 @@ def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kw __check_type(dataset.test.instances, np.ndarray, str) vectorizer = TfidfVectorizer(min_df=min_df, sublinear_tf=sublinear_tf, **kwargs) - training_documents = vectorizer.fit_transform(dataset.training.instances) - test_documents = vectorizer.transform(dataset.test.instances) - - if inplace: - dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.classes_) - dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.classes_) - dataset.vocabulary = vectorizer.vocabulary_ - return dataset - else: - training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.classes_) - test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.classes_) - return Dataset(training, test, vectorizer.vocabulary_) + return instance_transformation(dataset, vectorizer, inplace) def reduce_columns(dataset: Dataset, min_df=5, inplace=False): diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index 7f70bb8..07b2b1e 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -88,18 +88,30 @@ class WithConfidenceABC(ABC): METHODS = ['intervals', 'ellipse', 'ellipse-clr'] @abstractmethod - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): """ - Adds the method `quantify_conf` to the interface. This method returns not only the point-estimate, but + Adds the method `predict_conf` to the interface. This method returns not only the point-estimate, but also the confidence region around it. :param instances: a np.ndarray of shape (n_instances, n_features,) - :confidence_level: float in (0, 1) + :param confidence_level: float in (0, 1), default is 0.95 :return: a tuple (`point_estimate`, `conf_region`), where `point_estimate` is a np.ndarray of shape (n_classes,) and `conf_region` is an object from :class:`ConfidenceRegionABC` """ ... + def quantify_conf(self, instances, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): + """ + Alias to `predict_conf`. This method returns not only the point-estimate, but + also the confidence region around it. + + :param instances: a np.ndarray of shape (n_instances, n_features,) + :param confidence_level: float in (0, 1), default is 0.95 + :return: a tuple (`point_estimate`, `conf_region`), where `point_estimate` is a np.ndarray of shape + (n_classes,) and `conf_region` is an object from :class:`ConfidenceRegionABC` + """ + return self.predict_conf(instances=instances, confidence_level=confidence_level) + @classmethod def construct_region(cls, prev_estims, confidence_level=0.95, method='intervals'): """ @@ -227,6 +239,7 @@ class ConfidenceEllipseCLR(ConfidenceRegionABC): """ def __init__(self, X, confidence_level=0.95): + X = np.asarray(X) self.clr = CLRtransformation() Z = self.clr(X) self.mean_ = np.mean(X, axis=0) @@ -297,6 +310,9 @@ class ConfidenceIntervals(ConfidenceRegionABC): return proportion + def __repr__(self): + return '['+', '.join(f'({low:.4f}, {high:.4f})' for (low,high) in zip(self.I_low, self.I_high))+']' + class CLRtransformation: """ @@ -429,7 +445,7 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): self.aggregation_fit(classif_predictions, labels) return self - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): predictions = self.quantifier.classify(instances) return self.aggregate_conf(predictions, confidence_level=confidence_level) @@ -549,7 +565,7 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): samples = self.sample_from_posterior(classif_predictions)[_bayesian.P_TEST_Y] return np.asarray(samples.mean(axis=0), dtype=float) - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): classif_predictions = self.classify(instances) point_estimate = self.aggregate(classif_predictions) samples = self.get_prevalence_samples() # available after calling "aggregate" function diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index eff2283..5e84762 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -1,11 +1,14 @@ from typing import Union, Callable import numpy as np from sklearn.feature_extraction.text import CountVectorizer +from sklearn.utils import resample +from sklearn.preprocessing import normalize +from method.confidence import WithConfidenceABC, ConfidenceRegionABC from quapy.functional import get_divergence -from quapy.data import LabelledCollection from quapy.method.base import BaseQuantifier, BinaryQuantifier import quapy.functional as F +from scipy.optimize import lsq_linear class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier): @@ -149,53 +152,89 @@ class DMx(BaseQuantifier): return F.argmin_prevalence(loss, n_classes, method=self.search) -# class ReadMe(BaseQuantifier): -# -# def __init__(self, bootstrap_trials=100, bootstrap_range=100, bagging_trials=100, bagging_range=25, **vectorizer_kwargs): -# raise NotImplementedError('under development ...') -# self.bootstrap_trials = bootstrap_trials -# self.bootstrap_range = bootstrap_range -# self.bagging_trials = bagging_trials -# self.bagging_range = bagging_range -# self.vectorizer_kwargs = vectorizer_kwargs -# -# def fit(self, data: LabelledCollection): -# X, y = data.Xy -# self.vectorizer = CountVectorizer(binary=True, **self.vectorizer_kwargs) -# X = self.vectorizer.fit_transform(X) -# self.class_conditional_X = {i: X[y==i] for i in range(data.classes_)} -# -# def predict(self, X): -# X = self.vectorizer.transform(X) -# -# # number of features -# num_docs, num_feats = X.shape -# -# # bootstrap -# p_boots = [] -# for _ in range(self.bootstrap_trials): -# docs_idx = np.random.choice(num_docs, size=self.bootstra_range, replace=False) -# class_conditional_X = {i: X[docs_idx] for i, X in self.class_conditional_X.items()} -# Xboot = X[docs_idx] -# -# # bagging -# p_bags = [] -# for _ in range(self.bagging_trials): -# feat_idx = np.random.choice(num_feats, size=self.bagging_range, replace=False) -# class_conditional_Xbag = {i: X[:, feat_idx] for i, X in class_conditional_X.items()} -# Xbag = Xboot[:,feat_idx] -# p = self.std_constrained_linear_ls(Xbag, class_conditional_Xbag) -# p_bags.append(p) -# p_boots.append(np.mean(p_bags, axis=0)) -# -# p_mean = np.mean(p_boots, axis=0) -# p_std = np.std(p_bags, axis=0) -# -# return p_mean -# -# -# def std_constrained_linear_ls(self, X, class_cond_X: dict): -# pass +class ReadMe(BaseQuantifier, WithConfidenceABC): + + def __init__(self, + bootstrap_trials=100, + bagging_trials=100, + bagging_range=250, + confidence_level=0.95, + region='intervals', + random_state=None, + verbose=False): + self.bootstrap_trials = bootstrap_trials + self.bagging_trials = bagging_trials + self.bagging_range = bagging_range + self.confidence_level = confidence_level + self.region = region + self.random_state = random_state + self.verbose = verbose + + def fit(self, X, y): + self.rng = np.random.default_rng(self.random_state) + self.classes_ = np.unique(y) + n_features = X.shape[1] + + if self.bagging_range is None: + self.bagging_range = int(np.sqrt(n_features)) + + Xsize = X.shape[0] + + # Bootstrap loop + self.Xboots, self.yboots = [], [] + for _ in range(self.bootstrap_trials): + idx = self.rng.choice(Xsize, size=Xsize, replace=True) + self.Xboots.append(X[idx]) + self.yboots.append(y[idx]) + + return self + + def predict_conf(self, X, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): + from tqdm import tqdm + n_features = X.shape[1] + + boots_prevalences = [] + + for Xboots, yboots in tqdm( + zip(self.Xboots, self.yboots), + desc='bootstrap predictions', total=self.bootstrap_trials, disable=not self.verbose + ): + bagging_estimates = [] + for _ in range(self.bagging_trials): + feat_idx = self.rng.choice(n_features, size=self.bagging_range, replace=False) + Xboots_bagging = Xboots[:, feat_idx] + X_boots_bagging = X[:, feat_idx] + bagging_prev = self._quantify_iteration(Xboots_bagging, yboots, X_boots_bagging) + bagging_estimates.append(bagging_prev) + + boots_prevalences.append(np.mean(bagging_estimates, axis=0)) + + conf = WithConfidenceABC.construct_region(boots_prevalences, confidence_level, method=self.region) + prev_estim = conf.point_estimate() + + return prev_estim, conf + + + def predict(self, X): + prev_estim, _ = self.predict_conf(X) + return prev_estim + + + def _quantify_iteration(self, Xtr, ytr, Xte): + """Single ReadMe estimate.""" + n_classes = len(self.classes_) + PX_given_Y = np.zeros((n_classes, Xtr.shape[1])) + for i, c in enumerate(self.classes_): + PX_given_Y[i] = Xtr[ytr == c].sum(axis=0) + PX_given_Y = normalize(PX_given_Y, norm='l1', axis=1) + + PX = np.asarray(Xte.sum(axis=0)) + PX = normalize(PX, norm='l1', axis=1) + + res = lsq_linear(A=PX_given_Y.T, b=PX.ravel(), bounds=(0, 1)) + pY = np.maximum(res.x, 0) + return pY / pY.sum() + def _get_features_range(X): From 854b3ba3f941d973804d7b8adbc9d6a407e567b3 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Mon, 20 Oct 2025 18:33:45 +0200 Subject: [PATCH 05/43] documented ReadMe --- examples/18.ReadMe_for_text_analysis.py | 4 ++-- quapy/method/non_aggregative.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/18.ReadMe_for_text_analysis.py b/examples/18.ReadMe_for_text_analysis.py index d3e1c49..d885d0b 100644 --- a/examples/18.ReadMe_for_text_analysis.py +++ b/examples/18.ReadMe_for_text_analysis.py @@ -16,8 +16,8 @@ for test_prev in [[0.25, 0.75], [0.5, 0.5], [0.75, 0.25]]: prev_estim, conf = readme.predict_conf(sample.X) err = qp.error.mae(sample.prevalence(), prev_estim) print(f'true-prevalence={F.strprev(sample.prevalence())},\n' - f'predicted-prevalence={F.strprev(prev_estim)},\n' + f'predicted-prevalence={F.strprev(prev_estim)}, with confidence intervals {conf},\n' f'MAE={err:.4f}') - print(conf) + diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index 5e84762..0c2df5e 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -153,6 +153,30 @@ class DMx(BaseQuantifier): class ReadMe(BaseQuantifier, WithConfidenceABC): + """ + ReadMe is a non-aggregative quantification system proposed by + `Daniel Hopkins and Gary King, 2007. A method of automated nonparametric content analysis for + social science. American Journal of Political Science, 54(1):229–247. + `_. + The idea is to estimate `Q(Y=i)` directly from: + + :math:`Q(X)=\\sum_{i=1} Q(X|Y=i) Q(Y=i)` + + via least-squares regression, i.e., without incurring the cost of computing posterior probabilities. + However, this poses a very difficult representation in which the vector `Q(X)` and the matrix `Q(X|Y=i)` + can be of very high dimensions. In order to render the problem tracktable, ReadMe performs bagging in + the feature space. ReadMe also combines bagging with bootstrap in order to derive confidence intervals + around point estimations. + + :param bootstrap_trials: int, number of bootstrap trials (default 100) + :param bagging_trials: int, number of bagging trials (default 100) + :param bagging_range: int, number of features to keep for each bagging trial (default 250) + :param confidence_level: float, a value in (0,1) reflecting the desired confidence level (default 0.95) + :param region: str in 'intervals', 'ellipse', 'ellipse-clr'; indicates the preferred method for + defining the confidence region (see :class:`WithConfidenceABC`) + :param random_state: int or None, allows replicability (default None) + :param verbose: bool, whether to display information during the process (default False) + """ def __init__(self, bootstrap_trials=100, From c11b99e08ad00f82125c2bb9f1fbe388301d24b8 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Wed, 22 Oct 2025 18:51:35 +0200 Subject: [PATCH 06/43] improved ReadMe method --- CHANGE_LOG.txt | 1 + examples/18.ReadMe_for_text_analysis.py | 47 +++++++++-- quapy/data/preprocessing.py | 4 +- quapy/method/non_aggregative.py | 102 ++++++++++++++++++------ 4 files changed, 123 insertions(+), 31 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index 3e5155f..cc6be1b 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -2,6 +2,7 @@ Change Log 0.2.1 ----------------- - Improved documentation of confidence regions. +- Added ReadMe method by Daniel Hopkins and Gary King Change Log 0.2.0 ----------------- diff --git a/examples/18.ReadMe_for_text_analysis.py b/examples/18.ReadMe_for_text_analysis.py index d885d0b..7a70022 100644 --- a/examples/18.ReadMe_for_text_analysis.py +++ b/examples/18.ReadMe_for_text_analysis.py @@ -1,18 +1,55 @@ from sklearn.feature_extraction.text import CountVectorizer +from sklearn.feature_selection import SelectKBest, chi2 + import quapy as qp from quapy.method.non_aggregative import ReadMe import quapy.functional as F +from sklearn.pipeline import Pipeline +""" +This example showcases how to use the non-aggregative method ReadMe proposed by Hopkins and King. +This method is for text analysis, so let us first instantiate a dataset for sentiment quantification (we +use IMDb for this example). The method is quite computationally expensive, so we will restrict the training +set to 1000 documents only. +""" reviews = qp.datasets.fetch_reviews('imdb').reduce(n_train=1000, random_state=0) -encode_0_1 = CountVectorizer(min_df=5, binary=True) +""" +We need to convert text to bag-of-words representations. Actually, ReadMe requires the representations to be +binary (i.e., storing a 1 whenever a document contains certain word, or 0 otherwise), so we will not use +TFIDF weighting. We will also retain the top 1000 most important features according to chi2. +""" +encode_0_1 = Pipeline([ + ('0_1_terms', CountVectorizer(min_df=5, binary=True)), + ('feat_sel', SelectKBest(chi2, k=1000)) +]) train, test = qp.data.preprocessing.instance_transformation(reviews, encode_0_1, inplace=True).train_test -readme = ReadMe(bootstrap_trials=100, bagging_trials=100, bagging_range=100, random_state=0, verbose=True) -readme.fit(*train.Xy) +""" +We now instantiate ReadMe, with the prob_model='full' (default behaviour, implementing the Hopkins and King original +idea). This method consists of estimating Q(Y) by solving: -for test_prev in [[0.25, 0.75], [0.5, 0.5], [0.75, 0.25]]: - sample = reviews.test.sampling(500, *test_prev, random_state=0) +Q(X) = \sum_i Q(X|Y=i) Q(Y=i) + +without resorting to estimating the posteriors Q(Y=i|X), by solving a linear least-squares problem. +However, since Q(X) and Q(X|Y=i) are matrices of shape (2^K, 1) and (2^K, n), with K the number of features +and n the number of classes, their calculation becomes intractable. ReadMe instead performs bagging (i.e., it +samples small sets of features and averages the results) thus reducing K to a few terms. In our example we +set K (bagging_range) to 20, and the number of bagging_trials to 100. + +ReadMe also computes confidence intervals via bootstrap. We set the number of bootstrap trials to 100. +""" +readme = ReadMe(prob_model='full', bootstrap_trials=100, bagging_trials=100, bagging_range=20, random_state=0, verbose=True) +readme.fit(*train.Xy) # <- there is actually nothing happening here (only bootstrap resampling); the method is "lazy" + # and postpones most of the calculations to the test phase. + +# since the method is slow, we will only test 3 cases with different imbalances +few_negatives = [0.25, 0.75] +balanced = [0.5, 0.5] +few_positives = [0.75, 0.25] + +for test_prev in [few_negatives, balanced, few_positives]: + sample = reviews.test.sampling(500, *test_prev, random_state=0) # draw sets of 500 documents with desired prevs prev_estim, conf = readme.predict_conf(sample.X) err = qp.error.mae(sample.prevalence(), prev_estim) print(f'true-prevalence={F.strprev(sample.prevalence())},\n' diff --git a/quapy/data/preprocessing.py b/quapy/data/preprocessing.py index bc57d65..b196c11 100644 --- a/quapy/data/preprocessing.py +++ b/quapy/data/preprocessing.py @@ -22,8 +22,8 @@ def instance_transformation(dataset:Dataset, transformer, inplace=False): :return: a new :class:`quapy.data.base.Dataset` with transformed instances (if inplace=False) or a reference to the current Dataset (if inplace=True) where the instances have been transformed """ - training_transformed = transformer.fit_transform(dataset.training.instances) - test_transformed = transformer.transform(dataset.test.instances) + training_transformed = transformer.fit_transform(*dataset.training.Xy) + test_transformed = transformer.transform(dataset.test.X) if inplace: dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_) diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index 0c2df5e..ae894fd 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -1,4 +1,6 @@ -from typing import Union, Callable +from itertools import product +from tqdm import tqdm +from typing import Union, Callable, Counter import numpy as np from sklearn.feature_extraction.text import CountVectorizer from sklearn.utils import resample @@ -9,6 +11,7 @@ from quapy.functional import get_divergence from quapy.method.base import BaseQuantifier, BinaryQuantifier import quapy.functional as F from scipy.optimize import lsq_linear +from scipy import sparse class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier): @@ -152,6 +155,8 @@ class DMx(BaseQuantifier): return F.argmin_prevalence(loss, n_classes, method=self.search) + + class ReadMe(BaseQuantifier, WithConfidenceABC): """ ReadMe is a non-aggregative quantification system proposed by @@ -168,9 +173,21 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): the feature space. ReadMe also combines bagging with bootstrap in order to derive confidence intervals around point estimations. - :param bootstrap_trials: int, number of bootstrap trials (default 100) - :param bagging_trials: int, number of bagging trials (default 100) - :param bagging_range: int, number of features to keep for each bagging trial (default 250) + We use the same default parameters as in the official + `R implementation `_. + + :param prob_model: str ('naive', or 'full'), selects the modality in which the probabilities `Q(X)` and + `Q(X|Y)` are to be modelled. Options include "full", which corresponds to the original formulation of + ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which + `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where + `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes. + Of course, this approach is computationally prohibited for large `K`, so the computation is restricted to data + matrices with `K<=25` (although we recommend even smaller values of `K`). A much faster model is "naive", which + considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this + case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior). + :param bootstrap_trials: int, number of bootstrap trials (default 300) + :param bagging_trials: int, number of bagging trials (default 300) + :param bagging_range: int, number of features to keep for each bagging trial (default 15) :param confidence_level: float, a value in (0,1) reflecting the desired confidence level (default 0.95) :param region: str in 'intervals', 'ellipse', 'ellipse-clr'; indicates the preferred method for defining the confidence region (see :class:`WithConfidenceABC`) @@ -178,14 +195,21 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): :param verbose: bool, whether to display information during the process (default False) """ + MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION = 25 + PROBABILISTIC_MODELS = ["naive", "full"] + def __init__(self, - bootstrap_trials=100, - bagging_trials=100, - bagging_range=250, + prob_model="full", + bootstrap_trials=300, + bagging_trials=300, + bagging_range=15, confidence_level=0.95, region='intervals', random_state=None, verbose=False): + assert prob_model in ReadMe.PROBABILISTIC_MODELS, \ + f'unknown {prob_model=}, valid ones are {ReadMe.PROBABILISTIC_MODELS=}' + self.prob_model = prob_model self.bootstrap_trials = bootstrap_trials self.bagging_trials = bagging_trials self.bagging_range = bagging_range @@ -195,12 +219,11 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): self.verbose = verbose def fit(self, X, y): + self._check_matrix(X) + self.rng = np.random.default_rng(self.random_state) self.classes_ = np.unique(y) - n_features = X.shape[1] - if self.bagging_range is None: - self.bagging_range = int(np.sqrt(n_features)) Xsize = X.shape[0] @@ -214,11 +237,10 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): return self def predict_conf(self, X, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): - from tqdm import tqdm + self._check_matrix(X) + n_features = X.shape[1] - boots_prevalences = [] - for Xboots, yboots in tqdm( zip(self.Xboots, self.yboots), desc='bootstrap predictions', total=self.bootstrap_trials, disable=not self.verbose @@ -238,27 +260,59 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): return prev_estim, conf - def predict(self, X): prev_estim, _ = self.predict_conf(X) return prev_estim - def _quantify_iteration(self, Xtr, ytr, Xte): """Single ReadMe estimate.""" - n_classes = len(self.classes_) - PX_given_Y = np.zeros((n_classes, Xtr.shape[1])) - for i, c in enumerate(self.classes_): - PX_given_Y[i] = Xtr[ytr == c].sum(axis=0) - PX_given_Y = normalize(PX_given_Y, norm='l1', axis=1) + PX_given_Y = np.asarray([self._compute_P(Xtr[ytr == c]) for i,c in enumerate(self.classes_)]) + PX = self._compute_P(Xte) - PX = np.asarray(Xte.sum(axis=0)) - PX = normalize(PX, norm='l1', axis=1) - - res = lsq_linear(A=PX_given_Y.T, b=PX.ravel(), bounds=(0, 1)) + res = lsq_linear(A=PX_given_Y.T, b=PX, bounds=(0, 1)) pY = np.maximum(res.x, 0) return pY / pY.sum() + def _check_matrix(self, X): + """the "full" model requires estimating empirical distributions; due to the high computational cost, + this function is only made available for binary matrices""" + if self.prob_model == 'full' and not self._is_binary_matrix(X): + raise ValueError('the empirical distribution can only be computed efficiently on binary matrices') + + def _is_binary_matrix(self, X): + data = X.data if sparse.issparse(X) else X + return np.all((data == 0) | (data == 1)) + + def _compute_P(self, X): + if self.prob_model == 'naive': + return self._multinomial_distribution(X) + elif self.prob_model == 'full': + return self._empirical_distribution(X) + else: + raise ValueError(f'unknown {self.prob_model}; valid ones are {ReadMe.PROBABILISTIC_MODELS=}') + + def _empirical_distribution(self, X): + + if X.shape[1] > self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION: + raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions ' + f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}') + + # we convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5) + K = X.shape[1] + binary_powers = 1 << np.arange(K-1, -1, -1) # (2^K, ..., 32, 16, 8, 4, 2, 1) + X_as_binary_numbers = X @ binary_powers + + # count occurrences and compute probs + counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float) + probs = counts / counts.sum() + return probs + + def _multinomial_distribution(self, X): + PX = np.asarray(X.sum(axis=0)) + PX = normalize(PX, norm='l1', axis=1) + return PX.ravel() + + def _get_features_range(X): From 088ebcdd314cce768f0c01e356a4b52e145cadaf Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 23 Oct 2025 12:10:21 +0200 Subject: [PATCH 07/43] adding non aggregative methods experimental --- .../custom_vectorizers.py | 254 ++++++++++++++++++ experimental_non_aggregative/method_dxs.py | 148 ++++++++++ 2 files changed, 402 insertions(+) create mode 100644 experimental_non_aggregative/custom_vectorizers.py create mode 100644 experimental_non_aggregative/method_dxs.py diff --git a/experimental_non_aggregative/custom_vectorizers.py b/experimental_non_aggregative/custom_vectorizers.py new file mode 100644 index 0000000..13337b9 --- /dev/null +++ b/experimental_non_aggregative/custom_vectorizers.py @@ -0,0 +1,254 @@ +from scipy.sparse import csc_matrix, csr_matrix +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer, CountVectorizer +import numpy as np +from joblib import Parallel, delayed +import sklearn +import math +from scipy.stats import t + + +class ContTable: + def __init__(self, tp=0, tn=0, fp=0, fn=0): + self.tp=tp + self.tn=tn + self.fp=fp + self.fn=fn + + def get_d(self): return self.tp + self.tn + self.fp + self.fn + + def get_c(self): return self.tp + self.fn + + def get_not_c(self): return self.tn + self.fp + + def get_f(self): return self.tp + self.fp + + def get_not_f(self): return self.tn + self.fn + + def p_c(self): return (1.0*self.get_c())/self.get_d() + + def p_not_c(self): return 1.0-self.p_c() + + def p_f(self): return (1.0*self.get_f())/self.get_d() + + def p_not_f(self): return 1.0-self.p_f() + + def p_tp(self): return (1.0*self.tp) / self.get_d() + + def p_tn(self): return (1.0*self.tn) / self.get_d() + + def p_fp(self): return (1.0*self.fp) / self.get_d() + + def p_fn(self): return (1.0*self.fn) / self.get_d() + + def tpr(self): + c = 1.0*self.get_c() + return self.tp / c if c > 0.0 else 0.0 + + def fpr(self): + _c = 1.0*self.get_not_c() + return self.fp / _c if _c > 0.0 else 0.0 + + +def __ig_factor(p_tc, p_t, p_c): + den = p_t * p_c + if den != 0.0 and p_tc != 0: + return p_tc * math.log(p_tc / den, 2) + else: + return 0.0 + + +def information_gain(cell): + return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + \ + __ig_factor(cell.p_fp(), cell.p_f(), cell.p_not_c()) +\ + __ig_factor(cell.p_fn(), cell.p_not_f(), cell.p_c()) + \ + __ig_factor(cell.p_tn(), cell.p_not_f(), cell.p_not_c()) + + +def squared_information_gain(cell): + return information_gain(cell)**2 + + +def posneg_information_gain(cell): + ig = information_gain(cell) + if cell.tpr() < cell.fpr(): + return -ig + else: + return ig + + +def pos_information_gain(cell): + if cell.tpr() < cell.fpr(): + return 0 + else: + return information_gain(cell) + +def pointwise_mutual_information(cell): + return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + + +def gss(cell): + return cell.p_tp()*cell.p_tn() - cell.p_fp()*cell.p_fn() + + +def chi_square(cell): + den = cell.p_f() * cell.p_not_f() * cell.p_c() * cell.p_not_c() + if den==0.0: return 0.0 + num = gss(cell)**2 + return num / den + + +def conf_interval(xt, n): + if n>30: + z2 = 3.84145882069 # norm.ppf(0.5+0.95/2.0)**2 + else: + z2 = t.ppf(0.5 + 0.95 / 2.0, df=max(n-1,1)) ** 2 + p = (xt + 0.5 * z2) / (n + z2) + amplitude = 0.5 * z2 * math.sqrt((p * (1.0 - p)) / (n + z2)) + return p, amplitude + + +def strength(minPosRelFreq, minPos, maxNeg): + if minPos > maxNeg: + return math.log(2.0 * minPosRelFreq, 2.0) + else: + return 0.0 + + +#set cancel_features=True to allow some features to be weighted as 0 (as in the original article) +#however, for some extremely imbalanced dataset caused all documents to be 0 +def conf_weight(cell, cancel_features=False): + c = cell.get_c() + not_c = cell.get_not_c() + tp = cell.tp + fp = cell.fp + + pos_p, pos_amp = conf_interval(tp, c) + neg_p, neg_amp = conf_interval(fp, not_c) + + min_pos = pos_p-pos_amp + max_neg = neg_p+neg_amp + den = (min_pos + max_neg) + minpos_relfreq = min_pos / (den if den != 0 else 1) + + str_tplus = strength(minpos_relfreq, min_pos, max_neg); + + if str_tplus == 0 and not cancel_features: + return 1e-20 + + return str_tplus + + +def get_tsr_matrix(cell_matrix, tsr_score_funtion): + nC = len(cell_matrix) + nF = len(cell_matrix[0]) + tsr_matrix = [[tsr_score_funtion(cell_matrix[c,f]) for f in range(nF)] for c in range(nC)] + return np.array(tsr_matrix) + + +def feature_label_contingency_table(positive_document_indexes, feature_document_indexes, nD): + tp_ = len(positive_document_indexes & feature_document_indexes) + fp_ = len(feature_document_indexes - positive_document_indexes) + fn_ = len(positive_document_indexes - feature_document_indexes) + tn_ = nD - (tp_ + fp_ + fn_) + return ContTable(tp=tp_, tn=tn_, fp=fp_, fn=fn_) + + +def category_tables(feature_sets, category_sets, c, nD, nF): + return [feature_label_contingency_table(category_sets[c], feature_sets[f], nD) for f in range(nF)] + + +def get_supervised_matrix(coocurrence_matrix, label_matrix, n_jobs=-1): + """ + Computes the nC x nF supervised matrix M where Mcf is the 4-cell contingency table for feature f and class c. + Efficiency O(nF x nC x log(S)) where S is the sparse factor + """ + + nD, nF = coocurrence_matrix.shape + nD2, nC = label_matrix.shape + + if nD != nD2: + raise ValueError('Number of rows in coocurrence matrix shape %s and label matrix shape %s is not consistent' % + (coocurrence_matrix.shape,label_matrix.shape)) + + def nonzero_set(matrix, col): + return set(matrix[:, col].nonzero()[0]) + + if isinstance(coocurrence_matrix, csr_matrix): + coocurrence_matrix = csc_matrix(coocurrence_matrix) + feature_sets = [nonzero_set(coocurrence_matrix, f) for f in range(nF)] + category_sets = [nonzero_set(label_matrix, c) for c in range(nC)] + cell_matrix = Parallel(n_jobs=n_jobs, backend="threading")( + delayed(category_tables)(feature_sets, category_sets, c, nD, nF) for c in range(nC) + ) + return np.array(cell_matrix) + + +class TSRweighting(BaseEstimator,TransformerMixin): + """ + Supervised Term Weighting function based on any Term Selection Reduction (TSR) function (e.g., information gain, + chi-square, etc.) or, more generally, on any function that could be computed on the 4-cell contingency table for + each category-feature pair. + The supervised_4cell_matrix is a `(n_classes, n_words)` matrix containing the 4-cell contingency tables + for each class-word pair, and can be pre-computed (e.g., during the feature selection phase) and passed as an + argument. + When `n_classes>1`, i.e., in multiclass scenarios, a global_policy is used in order to determine a + single feature-score which informs about its relevance. Accepted policies include "max" (takes the max score + across categories), "ave" and "wave" (take the average, or weighted average, across all categories -- weights + correspond to the class prevalence), and "sum" (which sums all category scores). + """ + + def __init__(self, tsr_function, global_policy='max', supervised_4cell_matrix=None, sublinear_tf=True, norm='l2', min_df=3, n_jobs=-1): + if global_policy not in ['max', 'ave', 'wave', 'sum']: raise ValueError('Global policy should be in {"max", "ave", "wave", "sum"}') + self.tsr_function = tsr_function + self.global_policy = global_policy + self.supervised_4cell_matrix = supervised_4cell_matrix + self.sublinear_tf = sublinear_tf + self.norm = norm + self.min_df = min_df + self.n_jobs = n_jobs + + def fit(self, X, y): + self.count_vectorizer = CountVectorizer(min_df=self.min_df) + X = self.count_vectorizer.fit_transform(X) + + self.tf_vectorizer = TfidfTransformer( + norm=None, use_idf=False, smooth_idf=False, sublinear_tf=self.sublinear_tf + ).fit(X) + + if len(y.shape) == 1: + y = np.expand_dims(y, axis=1) + + nD, nC = y.shape + nF = len(self.tf_vectorizer.get_feature_names_out()) + + if self.supervised_4cell_matrix is None: + self.supervised_4cell_matrix = get_supervised_matrix(X, y, n_jobs=self.n_jobs) + else: + if self.supervised_4cell_matrix.shape != (nC, nF): + raise ValueError("Shape of supervised information matrix is inconsistent with X and y") + + tsr_matrix = get_tsr_matrix(self.supervised_4cell_matrix, self.tsr_function) + + if self.global_policy == 'ave': + self.global_tsr_vector = np.average(tsr_matrix, axis=0) + elif self.global_policy == 'wave': + category_prevalences = [sum(y[:,c])*1.0/nD for c in range(nC)] + self.global_tsr_vector = np.average(tsr_matrix, axis=0, weights=category_prevalences) + elif self.global_policy == 'sum': + self.global_tsr_vector = np.sum(tsr_matrix, axis=0) + elif self.global_policy == 'max': + self.global_tsr_vector = np.amax(tsr_matrix, axis=0) + return self + + def fit_transform(self, X, y): + return self.fit(X,y).transform(X) + + def transform(self, X): + if not hasattr(self, 'global_tsr_vector'): raise NameError('TSRweighting: transform method called before fit.') + X = self.count_vectorizer.transform(X) + tf_X = self.tf_vectorizer.transform(X).toarray() + weighted_X = np.multiply(tf_X, self.global_tsr_vector) + if self.norm is not None and self.norm!='none': + weighted_X = sklearn.preprocessing.normalize(weighted_X, norm=self.norm, axis=1, copy=False) + return csr_matrix(weighted_X) diff --git a/experimental_non_aggregative/method_dxs.py b/experimental_non_aggregative/method_dxs.py new file mode 100644 index 0000000..54986a2 --- /dev/null +++ b/experimental_non_aggregative/method_dxs.py @@ -0,0 +1,148 @@ +from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer +from sklearn.linear_model import LogisticRegression + +import quapy as qp +from data import LabelledCollection +import numpy as np + +from experimental_non_aggregative.custom_vectorizers import * +from protocol import APP +from quapy.method.aggregative import _get_divergence, HDy, DistributionMatching +from quapy.method.base import BaseQuantifier +from scipy import optimize +import pandas as pd + + +# TODO: explore the bernoulli (term presence/absence) variant +# TODO: explore the multinomial (term frequency) variant +# TODO: explore the multinomial + length normalization variant +# TODO: consolidate the TSR-variant (e.g., using information gain) variant; +# - works better with the idf? +# - works better with length normalization? +# - etc + +class DxS(BaseQuantifier): + def __init__(self, vectorizer=None, divergence='topsoe'): + self.vectorizer = vectorizer + self.divergence = divergence + + # def __as_distribution(self, instances): + # return np.asarray(instances.sum(axis=0) / instances.sum()).flatten() + + def __as_distribution(self, instances): + dist = instances.sum(axis=0) / instances.sum() + return np.asarray(dist).flatten() + + def fit(self, data: LabelledCollection): + + text_instances, labels = data.Xy + + if self.vectorizer is not None: + text_instances = self.vectorizer.fit_transform(text_instances, y=labels) + + distributions = [] + for class_i in data.classes_: + distributions.append(self.__as_distribution(text_instances[labels == class_i])) + self.validation_distribution = np.asarray(distributions) + return self + + def quantify(self, text_instances): + if self.vectorizer is not None: + text_instances = self.vectorizer.transform(text_instances) + + test_distribution = self.__as_distribution(text_instances) + divergence = _get_divergence(self.divergence) + n_classes, n_feats = self.validation_distribution.shape + + def match(prev): + prev = np.expand_dims(prev, axis=0) + mixture_distribution = (prev @ self.validation_distribution).flatten() + return divergence(test_distribution, mixture_distribution) + + # the initial point is set as the uniform distribution + uniform_distribution = np.full(fill_value=1 / n_classes, shape=(n_classes,)) + + # solutions are bounded to those contained in the unit-simplex + bounds = tuple((0, 1) for x in range(n_classes)) # values in [0,1] + constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)}) # values summing up to 1 + r = optimize.minimize(match, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints) + return r.x + + + +if __name__ == '__main__': + + qp.environ['SAMPLE_SIZE'] = 250 + qp.environ['N_JOBS'] = -1 + min_df = 10 + # dataset = 'imdb' + repeats = 10 + error = 'mae' + + div = 'topsoe' + + # generates tuples (dataset, method, method_name) + # (the dataset is needed for methods that process the dataset differently) + def gen_methods(): + + for dataset in qp.datasets.REVIEWS_SENTIMENT_DATASETS: + + data = qp.datasets.fetch_reviews(dataset, tfidf=False) + + bernoulli_vectorizer = CountVectorizer(min_df=min_df, binary=True) + dxs = DxS(divergence=div, vectorizer=bernoulli_vectorizer) + yield data, dxs, 'DxS-Bernoulli' + + multinomial_vectorizer = CountVectorizer(min_df=min_df, binary=False) + dxs = DxS(divergence=div, vectorizer=multinomial_vectorizer) + yield data, dxs, 'DxS-multinomial' + + tf_vectorizer = TfidfVectorizer(sublinear_tf=False, use_idf=False, min_df=min_df, norm=None) + dxs = DxS(divergence=div, vectorizer=tf_vectorizer) + yield data, dxs, 'DxS-TF' + + logtf_vectorizer = TfidfVectorizer(sublinear_tf=True, use_idf=False, min_df=min_df, norm=None) + dxs = DxS(divergence=div, vectorizer=logtf_vectorizer) + yield data, dxs, 'DxS-logTF' + + tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm=None) + dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + yield data, dxs, 'DxS-TFIDF' + + tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm='l2') + dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + yield data, dxs, 'DxS-TFIDF-l2' + + tsr_vectorizer = TSRweighting(tsr_function=information_gain, min_df=min_df, norm='l2') + dxs = DxS(divergence=div, vectorizer=tsr_vectorizer) + yield data, dxs, 'DxS-TFTSR-l2' + + data = qp.datasets.fetch_reviews(dataset, tfidf=True, min_df=min_df) + hdy = HDy(LogisticRegression()) + yield data, hdy, 'HDy' + + dm = DistributionMatching(LogisticRegression(), divergence=div, nbins=5) + yield data, dm, 'DM-5b' + + dm = DistributionMatching(LogisticRegression(), divergence=div, nbins=10) + yield data, dm, 'DM-10b' + + + result_path = 'results.csv' + with open(result_path, 'wt') as csv: + csv.write(f'Method\tDataset\tMAE\tMRAE\n') + for data, quantifier, quant_name in gen_methods(): + quantifier.fit(data.training) + report = qp.evaluation.evaluation_report(quantifier, APP(data.test, repeats=repeats), error_metrics=['mae','mrae'], verbose=True) + means = report.mean() + csv.write(f'{quant_name}\t{data.name}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\n') + + df = pd.read_csv(result_path, sep='\t') + # print(df) + + pv = df.pivot_table(index='Method', columns="Dataset", values=["MAE", "MRAE"]) + print(pv) + + + + From 41baeb78ca1918797a0115b3f43945ff07253258 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 23 Oct 2025 12:35:01 +0200 Subject: [PATCH 08/43] lazy index construction in labelled collection --- CHANGE_LOG.txt | 1 + experimental_non_aggregative/method_dxs.py | 24 ++++++++++++---------- quapy/data/base.py | 9 ++++++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index cc6be1b..b6066b9 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -3,6 +3,7 @@ Change Log 0.2.1 - Improved documentation of confidence regions. - Added ReadMe method by Daniel Hopkins and Gary King +- Internal index in LabelledCollection is now "lazy", and is only constructed if required. Change Log 0.2.0 ----------------- diff --git a/experimental_non_aggregative/method_dxs.py b/experimental_non_aggregative/method_dxs.py index 54986a2..c531118 100644 --- a/experimental_non_aggregative/method_dxs.py +++ b/experimental_non_aggregative/method_dxs.py @@ -7,7 +7,7 @@ import numpy as np from experimental_non_aggregative.custom_vectorizers import * from protocol import APP -from quapy.method.aggregative import _get_divergence, HDy, DistributionMatching +from quapy.method.aggregative import HDy, DistributionMatchingY from quapy.method.base import BaseQuantifier from scipy import optimize import pandas as pd @@ -30,28 +30,30 @@ class DxS(BaseQuantifier): # return np.asarray(instances.sum(axis=0) / instances.sum()).flatten() def __as_distribution(self, instances): - dist = instances.sum(axis=0) / instances.sum() + dist = instances.mean(axis=0) return np.asarray(dist).flatten() - def fit(self, data: LabelledCollection): + def fit(self, text_instances, labels): - text_instances, labels = data.Xy + classes = np.unique(labels) if self.vectorizer is not None: text_instances = self.vectorizer.fit_transform(text_instances, y=labels) distributions = [] - for class_i in data.classes_: + for class_i in classes: distributions.append(self.__as_distribution(text_instances[labels == class_i])) + self.validation_distribution = np.asarray(distributions) + return self - def quantify(self, text_instances): + def predict(self, text_instances): if self.vectorizer is not None: text_instances = self.vectorizer.transform(text_instances) test_distribution = self.__as_distribution(text_instances) - divergence = _get_divergence(self.divergence) + divergence = qp.functional.get_divergence(self.divergence) n_classes, n_feats = self.validation_distribution.shape def match(prev): @@ -121,10 +123,10 @@ if __name__ == '__main__': hdy = HDy(LogisticRegression()) yield data, hdy, 'HDy' - dm = DistributionMatching(LogisticRegression(), divergence=div, nbins=5) + dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=5) yield data, dm, 'DM-5b' - dm = DistributionMatching(LogisticRegression(), divergence=div, nbins=10) + dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=10) yield data, dm, 'DM-10b' @@ -132,9 +134,9 @@ if __name__ == '__main__': with open(result_path, 'wt') as csv: csv.write(f'Method\tDataset\tMAE\tMRAE\n') for data, quantifier, quant_name in gen_methods(): - quantifier.fit(data.training) + quantifier.fit(*data.training.Xy) report = qp.evaluation.evaluation_report(quantifier, APP(data.test, repeats=repeats), error_metrics=['mae','mrae'], verbose=True) - means = report.mean() + means = report.mean(numeric_only=True) csv.write(f'{quant_name}\t{data.name}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\n') df = pd.read_csv(result_path, sep='\t') diff --git a/quapy/data/base.py b/quapy/data/base.py index 82c57db..6db182a 100644 --- a/quapy/data/base.py +++ b/quapy/data/base.py @@ -33,7 +33,6 @@ class LabelledCollection: else: self.instances = np.asarray(instances) self.labels = np.asarray(labels) - n_docs = len(self) if classes is None: self.classes_ = F.classes_from_labels(self.labels) else: @@ -41,7 +40,13 @@ class LabelledCollection: self.classes_.sort() if len(set(self.labels).difference(set(classes))) > 0: raise ValueError(f'labels ({set(self.labels)}) contain values not included in classes_ ({set(classes)})') - self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_} + self._index = None + + @property + def index(self): + if self._index is None: + self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_} + return self._index @classmethod def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs): From f227ed2f6024edafafad7ef4f83ba9a53f470a32 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 23 Oct 2025 14:12:39 +0200 Subject: [PATCH 09/43] adding kdex --- experimental_non_aggregative/method_dxs.py | 112 ++++++++++++++++----- quapy/data/preprocessing.py | 7 +- 2 files changed, 89 insertions(+), 30 deletions(-) diff --git a/experimental_non_aggregative/method_dxs.py b/experimental_non_aggregative/method_dxs.py index c531118..93fb67e 100644 --- a/experimental_non_aggregative/method_dxs.py +++ b/experimental_non_aggregative/method_dxs.py @@ -1,16 +1,21 @@ +from scipy.sparse import issparse +from sklearn.decomposition import TruncatedSVD from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer from sklearn.linear_model import LogisticRegression +from sklearn.preprocessing import StandardScaler import quapy as qp from data import LabelledCollection import numpy as np from experimental_non_aggregative.custom_vectorizers import * +from method._kdey import KDEBase from protocol import APP from quapy.method.aggregative import HDy, DistributionMatchingY from quapy.method.base import BaseQuantifier from scipy import optimize import pandas as pd +import quapy.functional as F # TODO: explore the bernoulli (term presence/absence) variant @@ -72,6 +77,51 @@ class DxS(BaseQuantifier): +class KDExML(BaseQuantifier, KDEBase): + + def __init__(self, bandwidth=0.1, standardize=False): + self._check_bandwidth(bandwidth) + self.bandwidth = bandwidth + self.standardize = standardize + + def fit(self, X, y): + classes = sorted(np.unique(y)) + + if self.standardize: + self.scaler = StandardScaler() + X = self.scaler.fit_transform(X) + + if issparse(X): + X = X.toarray() + + self.mix_densities = self.get_mixture_components(X, y, classes, self.bandwidth) + return self + + def predict(self, X): + """ + Searches for the mixture model parameter (the sought prevalence values) that maximizes the likelihood + of the data (i.e., that minimizes the negative log-likelihood) + + :param X: instances in the sample + :return: a vector of class prevalence estimates + """ + epsilon = 1e-10 + if issparse(X): + X = X.toarray() + n_classes = len(self.mix_densities) + if self.standardize: + X = self.scaler.transform(X) + test_densities = [self.pdf(kde_i, X) for kde_i in self.mix_densities] + + def neg_loglikelihood(prev): + test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) + test_loglikelihood = np.log(test_mixture_likelihood + epsilon) + return -np.sum(test_loglikelihood) + + return F.optim_minimize(neg_loglikelihood, n_classes) + + + if __name__ == '__main__': qp.environ['SAMPLE_SIZE'] = 250 @@ -91,43 +141,51 @@ if __name__ == '__main__': data = qp.datasets.fetch_reviews(dataset, tfidf=False) - bernoulli_vectorizer = CountVectorizer(min_df=min_df, binary=True) - dxs = DxS(divergence=div, vectorizer=bernoulli_vectorizer) - yield data, dxs, 'DxS-Bernoulli' - - multinomial_vectorizer = CountVectorizer(min_df=min_df, binary=False) - dxs = DxS(divergence=div, vectorizer=multinomial_vectorizer) - yield data, dxs, 'DxS-multinomial' - - tf_vectorizer = TfidfVectorizer(sublinear_tf=False, use_idf=False, min_df=min_df, norm=None) - dxs = DxS(divergence=div, vectorizer=tf_vectorizer) - yield data, dxs, 'DxS-TF' - - logtf_vectorizer = TfidfVectorizer(sublinear_tf=True, use_idf=False, min_df=min_df, norm=None) - dxs = DxS(divergence=div, vectorizer=logtf_vectorizer) - yield data, dxs, 'DxS-logTF' - - tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm=None) - dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) - yield data, dxs, 'DxS-TFIDF' - - tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm='l2') - dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) - yield data, dxs, 'DxS-TFIDF-l2' + # bernoulli_vectorizer = CountVectorizer(min_df=min_df, binary=True) + # dxs = DxS(divergence=div, vectorizer=bernoulli_vectorizer) + # yield data, dxs, 'DxS-Bernoulli' + # + # multinomial_vectorizer = CountVectorizer(min_df=min_df, binary=False) + # dxs = DxS(divergence=div, vectorizer=multinomial_vectorizer) + # yield data, dxs, 'DxS-multinomial' + # + # tf_vectorizer = TfidfVectorizer(sublinear_tf=False, use_idf=False, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=tf_vectorizer) + # yield data, dxs, 'DxS-TF' + # + # logtf_vectorizer = TfidfVectorizer(sublinear_tf=True, use_idf=False, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=logtf_vectorizer) + # yield data, dxs, 'DxS-logTF' + # + # tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + # yield data, dxs, 'DxS-TFIDF' + # + # tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm='l2') + # dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + # yield data, dxs, 'DxS-TFIDF-l2' tsr_vectorizer = TSRweighting(tsr_function=information_gain, min_df=min_df, norm='l2') dxs = DxS(divergence=div, vectorizer=tsr_vectorizer) yield data, dxs, 'DxS-TFTSR-l2' data = qp.datasets.fetch_reviews(dataset, tfidf=True, min_df=min_df) + + kdex = KDExML() + reduction = TruncatedSVD(n_components=100, random_state=0) + red_data = qp.data.preprocessing.instance_transformation(data, transformer=reduction, inplace=False) + yield red_data, kdex, 'KDEx' + hdy = HDy(LogisticRegression()) yield data, hdy, 'HDy' - dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=5) - yield data, dm, 'DM-5b' + # dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=5) + # yield data, dm, 'DM-5b' + # + # dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=10) + # yield data, dm, 'DM-10b' + - dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=10) - yield data, dm, 'DM-10b' result_path = 'results.csv' diff --git a/quapy/data/preprocessing.py b/quapy/data/preprocessing.py index b196c11..5f7e0a9 100644 --- a/quapy/data/preprocessing.py +++ b/quapy/data/preprocessing.py @@ -24,6 +24,7 @@ def instance_transformation(dataset:Dataset, transformer, inplace=False): """ training_transformed = transformer.fit_transform(*dataset.training.Xy) test_transformed = transformer.transform(dataset.test.X) + orig_name = dataset.name if inplace: dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_) @@ -34,10 +35,10 @@ def instance_transformation(dataset:Dataset, transformer, inplace=False): else: training = LabelledCollection(training_transformed, dataset.training.labels.copy(), dataset.classes_) test = LabelledCollection(test_transformed, dataset.test.labels.copy(), dataset.classes_) + vocab = None if hasattr(transformer, 'vocabulary_'): - return Dataset(training, test, transformer.vocabulary_) - else: - return Dataset(training, test) + vocab = transformer.vocabulary_ + return Dataset(training, test, vocabulary=vocab, name=orig_name) def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs): From 3c09b1c98a5af2965889982b50f8093cfe682b53 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Thu, 13 Nov 2025 18:45:07 +0100 Subject: [PATCH 10/43] adding prev@densities in KDEyML, huge effiency improvement... --- quapy/method/_kdey.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quapy/method/_kdey.py b/quapy/method/_kdey.py index 88613c2..f004c1a 100644 --- a/quapy/method/_kdey.py +++ b/quapy/method/_kdey.py @@ -134,7 +134,8 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): test_densities = [self.pdf(kde_i, posteriors) for kde_i in self.mix_densities] def neg_loglikelihood(prev): - test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) + # test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) + test_mixture_likelihood = prev @ test_densities test_loglikelihood = np.log(test_mixture_likelihood + epsilon) return -np.sum(test_loglikelihood) From a868d2d56189b6e7c6429ee6c6014d6dafad6efc Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Fri, 14 Nov 2025 16:10:17 +0100 Subject: [PATCH 11/43] import fix --- quapy/method/non_aggregative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index ae894fd..6f204e4 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -6,7 +6,7 @@ from sklearn.feature_extraction.text import CountVectorizer from sklearn.utils import resample from sklearn.preprocessing import normalize -from method.confidence import WithConfidenceABC, ConfidenceRegionABC +from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC from quapy.functional import get_divergence from quapy.method.base import BaseQuantifier, BinaryQuantifier import quapy.functional as F From 3268e9fada0af5ff2206d0df142de84b754737f2 Mon Sep 17 00:00:00 2001 From: pglez82 Date: Fri, 14 Nov 2025 18:35:40 +0100 Subject: [PATCH 12/43] PQ (precise quantifier) --- quapy/method/__init__.py | 2 + quapy/method/_bayesian.py | 52 +++++++++++++++++++ quapy/method/confidence.py | 104 ++++++++++++++++++++++++++++++++++++- quapy/method/stan/pq.stan | 38 ++++++++++++++ 4 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 quapy/method/stan/pq.stan diff --git a/quapy/method/__init__.py b/quapy/method/__init__.py index 4c2ec1c..ab7a59b 100644 --- a/quapy/method/__init__.py +++ b/quapy/method/__init__.py @@ -29,6 +29,7 @@ AGGREGATIVE_METHODS = { aggregative.KDEyHD, # aggregative.OneVsAllAggregative, confidence.BayesianCC, + confidence.PQ, } BINARY_METHODS = { @@ -40,6 +41,7 @@ BINARY_METHODS = { aggregative.MAX, aggregative.MS, aggregative.MS2, + confidence.PQ, } MULTICLASS_METHODS = { diff --git a/quapy/method/_bayesian.py b/quapy/method/_bayesian.py index c783f10..c5feaf3 100644 --- a/quapy/method/_bayesian.py +++ b/quapy/method/_bayesian.py @@ -8,6 +8,7 @@ try: import jax.numpy as jnp import numpyro import numpyro.distributions as dist + import stan DEPENDENCIES_INSTALLED = True except ImportError: @@ -15,6 +16,7 @@ except ImportError: jnp = None numpyro = None dist = None + stan = None DEPENDENCIES_INSTALLED = False @@ -77,3 +79,53 @@ def sample_posterior( rng_key = jax.random.PRNGKey(seed) mcmc.run(rng_key, n_c_unlabeled=n_c_unlabeled, n_y_and_c_labeled=n_y_and_c_labeled) return mcmc.get_samples() + + + +def pq_stan(stan_code, n_bins, pos_hist, neg_hist, test_hist, number_of_samples, num_warmup, stan_seed): + """ + Perform Bayesian prevalence estimation using a Stan model for probabilistic quantification. + + This function builds and samples from a Stan model that implements a bin-based Bayesian + quantifier. It uses the class-conditional histograms of the classifier + outputs for positive and negative examples, along with the test histogram, to estimate + the posterior distribution of prevalence in the test set. + + Parameters + ---------- + stan_code : str + The Stan model code as a string. + n_bins : int + Number of bins used to build the histograms for positive and negative examples. + pos_hist : array-like of shape (n_bins,) + Histogram counts of the classifier outputs for the positive class. + neg_hist : array-like of shape (n_bins,) + Histogram counts of the classifier outputs for the negative class. + test_hist : array-like of shape (n_bins,) + Histogram counts of the classifier outputs for the test set, binned using the same bins. + number_of_samples : int + Number of post-warmup samples to draw from the Stan posterior. + num_warmup : int + Number of warmup iterations for the sampler. + stan_seed : int + Random seed for Stan model compilation and sampling, ensuring reproducibility. + + Returns + ------- + prev_samples : numpy.ndarray + An array of posterior samples of the prevalence (`prev`) in the test set. + Each element corresponds to one draw from the posterior distribution. + """ + + stan_data = { + 'n_bucket': n_bins, + 'train_neg': neg_hist.tolist(), + 'train_pos': pos_hist.tolist(), + 'test': test_hist.tolist(), + 'posterior': 1 + } + + stan_model = stan.build(stan_code, data=stan_data, random_seed=stan_seed) + fit = stan_model.sample(num_chains=1, num_samples=number_of_samples,num_warmup=num_warmup) + + return fit['prev'] diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index 07b2b1e..dd9e05f 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -5,9 +5,8 @@ from sklearn.metrics import confusion_matrix import quapy as qp import quapy.functional as F from quapy.method import _bayesian -from quapy.method.aggregative import AggregativeCrispQuantifier from quapy.data import LabelledCollection -from quapy.method.aggregative import AggregativeQuantifier +from quapy.method.aggregative import AggregativeQuantifier, AggregativeCrispQuantifier, AggregativeSoftQuantifier, BinaryAggregativeQuantifier from scipy.stats import chi2 from sklearn.utils import resample from abc import ABC, abstractmethod @@ -571,3 +570,104 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): samples = self.get_prevalence_samples() # available after calling "aggregate" function region = WithConfidenceABC.construct_region(samples, confidence_level=self.confidence_level, method=self.region) return point_estimate, region + + +class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): + """ + `Precise Quantifier: Bayesian distribution matching quantifier , + which is a variant of :class:`HDy` that calculates the posterior probability distribution + over the prevalence vectors, rather than providing a point estimate. + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be + the one indicated in `qp.environ['DEFAULT_CLS']` + :param val_split: specifies the data used for generating classifier predictions. This specification + can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to + be extracted from the training set; or as an integer (default 5), indicating that the predictions + are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value + for `k`); or as a tuple `(X,y)` defining the specific set of data to use for validation. Set to + None when the method does not require any validation data, in order to avoid that some portion of + the training data be wasted. + :param num_warmup: number of warmup iterations for the STAN sampler (default 500) + :param num_samples: number of samples to draw from the posterior (default 1000) + :param stan_seed: random seed for the STAN sampler (default 0) + :param region: string, set to `intervals` for constructing confidence intervals (default), or to + `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for + constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + """ + def __init__(self, + classifier: BaseEstimator=None, + fit_classifier=True, + val_split: int = 5, + n_bins: int = 4, + fixed_bins: bool = False, + num_warmup: int = 500, + num_samples: int = 1_000, + stan_seed: int = 0, + region: str = 'intervals'): + + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + + if _bayesian.DEPENDENCIES_INSTALLED is False: + raise ImportError("Auxiliary dependencies are required. " + "Run `$ pip install quapy[bayes]` to install them.") + + super().__init__(classifier, fit_classifier, val_split) + self.n_bins = n_bins + self.fixed_bins = fixed_bins + self.num_warmup = num_warmup + self.num_samples = num_samples + self.region = region + self.stan_seed = stan_seed + with open('quapy/method/stan/pq.stan', 'r') as f: + self.stan_code = str(f.read()) + + def aggregation_fit(self, classif_predictions, labels): + y_pred = classif_predictions[:, self.pos_label] + + # Compute bin limits + if self.fixed_bins: + # Uniform bins in [0,1] + self.bin_limits = np.linspace(0, 1, self.n_bins + 1) + else: + # Quantile bins + self.bin_limits = np.quantile(y_pred, np.linspace(0, 1, self.n_bins + 1)) + + # Assign each prediction to a bin + bin_indices = np.digitize(y_pred, self.bin_limits[1:-1], right=True) + + # Positive and negative masks + pos_mask = (labels == self.pos_label) + neg_mask = ~pos_mask + + # Count positives and negatives per bin + self.pos_hist = np.bincount(bin_indices[pos_mask], minlength=self.n_bins) + self.neg_hist = np.bincount(bin_indices[neg_mask], minlength=self.n_bins) + + + def aggregate(self, classif_predictions): + Px_test = classif_predictions[:, self.pos_label] + test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) + self.prev_distribution = _bayesian.pq_stan(self.stan_code, + self.n_bins, + self.pos_hist, + self.neg_hist, + test_hist, + self.num_samples, + self.num_warmup, + self.stan_seed) + return F.as_binary_prevalence(self.prev_distribution.mean()) + + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + classif_predictions = self.classify(instances) + point_estimate = self.aggregate(classif_predictions) + samples = self.prev_distribution + region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region) + return point_estimate, region + diff --git a/quapy/method/stan/pq.stan b/quapy/method/stan/pq.stan new file mode 100644 index 0000000..b759ddf --- /dev/null +++ b/quapy/method/stan/pq.stan @@ -0,0 +1,38 @@ +data { + int n_bucket; + array[n_bucket] int train_pos; + array[n_bucket] int train_neg; + array[n_bucket] int test; + int posterior; +} + +transformed data{ + row_vector[n_bucket] train_pos_rv; + row_vector[n_bucket] train_neg_rv; + row_vector[n_bucket] test_rv; + real n_test; + + train_pos_rv = to_row_vector( train_pos ); + train_neg_rv = to_row_vector( train_neg ); + test_rv = to_row_vector( test ); + n_test = sum( test ); +} + +parameters { + simplex[n_bucket] p_neg; + simplex[n_bucket] p_pos; + real prev_prior; +} + +model { + if( posterior ) { + target += train_neg_rv * log( p_neg ); + target += train_pos_rv * log( p_pos ); + target += test_rv * log( p_neg * ( 1 - prev_prior) + p_pos * prev_prior ); + } +} + +generated quantities { + real prev; + prev = sum( binomial_rng(test, 1 / ( 1 + (p_neg./p_pos) *(1-prev_prior)/prev_prior ) ) ) / n_test; +} \ No newline at end of file From e4c07e18353e6d147bb30b17c9c79edb21608a5e Mon Sep 17 00:00:00 2001 From: pglez82 Date: Sat, 15 Nov 2025 16:51:48 +0100 Subject: [PATCH 13/43] changing the way the file is loaded --- quapy/method/_bayesian.py | 4 ++++ quapy/method/confidence.py | 3 +-- setup.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/quapy/method/_bayesian.py b/quapy/method/_bayesian.py index c5feaf3..da65eed 100644 --- a/quapy/method/_bayesian.py +++ b/quapy/method/_bayesian.py @@ -2,6 +2,7 @@ Utility functions for `Bayesian quantification `_ methods. """ import numpy as np +import importlib.resources try: import jax @@ -82,6 +83,9 @@ def sample_posterior( +def load_stan_file(): + return importlib.resources.files('quapy.method').joinpath('stan/pq.stan').read_text(encoding='utf-8') + def pq_stan(stan_code, n_bins, pos_hist, neg_hist, test_hist, number_of_samples, num_warmup, stan_seed): """ Perform Bayesian prevalence estimation using a Stan model for probabilistic quantification. diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index dd9e05f..ab649c2 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -624,8 +624,7 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.num_samples = num_samples self.region = region self.stan_seed = stan_seed - with open('quapy/method/stan/pq.stan', 'r') as f: - self.stan_code = str(f.read()) + self.stan_code = _bayesian.load_stan_file() def aggregation_fit(self, classif_predictions, labels): y_pred = classif_predictions[:, self.pos_label] diff --git a/setup.py b/setup.py index ba5f205..c058f9d 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ setup( # Similar to `install_requires` above, these must be valid existing # projects. extras_require={ # Optional - 'bayes': ['jax', 'jaxlib', 'numpyro'], + 'bayes': ['jax', 'jaxlib', 'numpyro', 'pystan'], 'neural': ['torch'], 'tests': ['certifi'], 'docs' : ['sphinx-rtd-theme', 'myst-parser'], From c6492a0f205518130f81c5befd962a17ee4eb9b6 Mon Sep 17 00:00:00 2001 From: pglez82 Date: Sat, 15 Nov 2025 17:04:44 +0100 Subject: [PATCH 14/43] fixing import --- quapy/method/non_aggregative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index ae894fd..6f204e4 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -6,7 +6,7 @@ from sklearn.feature_extraction.text import CountVectorizer from sklearn.utils import resample from sklearn.preprocessing import normalize -from method.confidence import WithConfidenceABC, ConfidenceRegionABC +from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC from quapy.functional import get_divergence from quapy.method.base import BaseQuantifier, BinaryQuantifier import quapy.functional as F From 46e7246f3a4f0a8256af5f6b9b61db3f49a08e77 Mon Sep 17 00:00:00 2001 From: pglez82 Date: Sat, 15 Nov 2025 17:04:55 +0100 Subject: [PATCH 15/43] adding stan file to setup --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index c058f9d..2464122 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,12 @@ setup( # packages=find_packages(include=['quapy', 'quapy.*']), # Required + package_data={ + # For the 'quapy.method' package, include all files + # in the 'stan' subdirectory that end with .stan + 'quapy.method': ['stan/*.stan'] + }, + python_requires='>=3.8, <4', install_requires=['scikit-learn', 'pandas', 'tqdm', 'matplotlib', 'joblib', 'xlrd', 'abstention', 'ucimlrepo', 'certifi'], From d9cf6cc11d4c0f30867578f374e3695a1c8f0882 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Sat, 15 Nov 2025 17:56:37 +0100 Subject: [PATCH 16/43] index in labelled collection from versions restored --- quapy/data/base.py | 2 +- quapy/method/confidence.py | 24 ++++++++++-------------- quapy/method/stan/pq.stan | 3 ++- setup.py | 2 +- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/quapy/data/base.py b/quapy/data/base.py index 6db182a..9bdf135 100644 --- a/quapy/data/base.py +++ b/quapy/data/base.py @@ -44,7 +44,7 @@ class LabelledCollection: @property def index(self): - if self._index is None: + if not hasattr(self, '_index') or self._index is None: self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_} return self._index diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index dd9e05f..7423545 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -13,6 +13,7 @@ from abc import ABC, abstractmethod from scipy.special import softmax, factorial import copy from functools import lru_cache +from pathlib import Path """ This module provides implementation of different types of confidence regions, and the implementation of Bootstrap @@ -613,7 +614,7 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): if num_samples <= 0: raise ValueError(f'parameter {num_samples=} must be a positive integer') - if _bayesian.DEPENDENCIES_INSTALLED is False: + if not _bayesian.DEPENDENCIES_INSTALLED: raise ImportError("Auxiliary dependencies are required. " "Run `$ pip install quapy[bayes]` to install them.") @@ -624,7 +625,9 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.num_samples = num_samples self.region = region self.stan_seed = stan_seed - with open('quapy/method/stan/pq.stan', 'r') as f: + # with open('quapy/method/stan/pq.stan', 'r') as f: + stan_path = Path(__file__).resolve().parent / "stan" / "pq.stan" + with stan_path.open("r") as f: self.stan_code = str(f.read()) def aggregation_fit(self, classif_predictions, labels): @@ -649,24 +652,17 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.pos_hist = np.bincount(bin_indices[pos_mask], minlength=self.n_bins) self.neg_hist = np.bincount(bin_indices[neg_mask], minlength=self.n_bins) - def aggregate(self, classif_predictions): Px_test = classif_predictions[:, self.pos_label] test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) - self.prev_distribution = _bayesian.pq_stan(self.stan_code, - self.n_bins, - self.pos_hist, - self.neg_hist, - test_hist, - self.num_samples, - self.num_warmup, - self.stan_seed) + self.prev_distribution = _bayesian.pq_stan( + self.stan_code, self.n_bins, self.pos_hist, self.neg_hist, test_hist, + self.num_samples, self.num_warmup, self.stan_seed + ) return F.as_binary_prevalence(self.prev_distribution.mean()) - def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): - classif_predictions = self.classify(instances) - point_estimate = self.aggregate(classif_predictions) + point_estimate = self.predict(instances) samples = self.prev_distribution region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region) return point_estimate, region diff --git a/quapy/method/stan/pq.stan b/quapy/method/stan/pq.stan index b759ddf..a5bc52a 100644 --- a/quapy/method/stan/pq.stan +++ b/quapy/method/stan/pq.stan @@ -35,4 +35,5 @@ model { generated quantities { real prev; prev = sum( binomial_rng(test, 1 / ( 1 + (p_neg./p_pos) *(1-prev_prior)/prev_prior ) ) ) / n_test; -} \ No newline at end of file +} + diff --git a/setup.py b/setup.py index ba5f205..c058f9d 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ setup( # Similar to `install_requires` above, these must be valid existing # projects. extras_require={ # Optional - 'bayes': ['jax', 'jaxlib', 'numpyro'], + 'bayes': ['jax', 'jaxlib', 'numpyro', 'pystan'], 'neural': ['torch'], 'tests': ['certifi'], 'docs' : ['sphinx-rtd-theme', 'myst-parser'], From 047cb9e533ed24cb05544a83df1feaf1372b1cfb Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Sat, 15 Nov 2025 18:54:04 +0100 Subject: [PATCH 17/43] merged --- quapy/method/confidence.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index 3adcf38..fd8c84d 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -565,10 +565,12 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): return np.asarray(samples.mean(axis=0), dtype=float) def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + if confidence_level is None: + confidence_level = self.confidence_level classif_predictions = self.classify(instances) point_estimate = self.aggregate(classif_predictions) samples = self.get_prevalence_samples() # available after calling "aggregate" function - region = WithConfidenceABC.construct_region(samples, confidence_level=self.confidence_level, method=self.region) + region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region) return point_estimate, region @@ -606,6 +608,7 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): num_warmup: int = 500, num_samples: int = 1_000, stan_seed: int = 0, + confidence_level: float = 0.95, region: str = 'intervals'): if num_warmup <= 0: @@ -622,9 +625,10 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.fixed_bins = fixed_bins self.num_warmup = num_warmup self.num_samples = num_samples - self.region = region self.stan_seed = stan_seed self.stan_code = _bayesian.load_stan_file() + self.confidence_level = confidence_level + self.region = region def aggregation_fit(self, classif_predictions, labels): y_pred = classif_predictions[:, self.pos_label] @@ -651,16 +655,23 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): def aggregate(self, classif_predictions): Px_test = classif_predictions[:, self.pos_label] test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) - self.prev_distribution = _bayesian.pq_stan( + prevs = _bayesian.pq_stan( self.stan_code, self.n_bins, self.pos_hist, self.neg_hist, test_hist, self.num_samples, self.num_warmup, self.stan_seed - ) - return F.as_binary_prevalence(self.prev_distribution.mean()) + ).flatten() + self.prev_distribution = np.vstack([1-prevs, prevs]).T + return self.prev_distribution.mean(axis=0) - def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): - classif_predictions = self.classify(instances) - point_estimate = self.aggregate(classif_predictions) + def aggregate_conf(self, predictions, confidence_level=None): + if confidence_level is None: + confidence_level = self.confidence_level + point_estimate = self.aggregate(predictions) samples = self.prev_distribution region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region) return point_estimate, region + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + predictions = self.classify(instances) + return self.aggregate_conf(predictions, confidence_level=confidence_level) + + From 9da4fd57db8e9d43182d9f579a035a8b60aa965c Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Mon, 17 Nov 2025 17:53:56 +0100 Subject: [PATCH 18/43] added property samples to confidence regions --- quapy/method/confidence.py | 53 ++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index 07b2b1e..c9e24c4 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -80,6 +80,12 @@ class ConfidenceRegionABC(ABC): proportion = np.clip(self.coverage(uniform_simplex), 0., 1.) return proportion + @property + @abstractmethod + def samples(self): + """ Returns internal samples """ + ... + class WithConfidenceABC(ABC): """ @@ -185,30 +191,35 @@ class ConfidenceEllipseSimplex(ConfidenceRegionABC): """ Instantiates a Confidence Ellipse in the probability simplex. - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) :param confidence_level: float, the confidence level (default 0.95) """ - def __init__(self, X, confidence_level=0.95): + def __init__(self, samples, confidence_level=0.95): assert 0. < confidence_level < 1., f'{confidence_level=} must be in range(0,1)' - X = np.asarray(X) + samples = np.asarray(samples) - self.mean_ = X.mean(axis=0) - self.cov_ = np.cov(X, rowvar=False, ddof=1) + self.mean_ = samples.mean(axis=0) + self.cov_ = np.cov(samples, rowvar=False, ddof=1) try: self.precision_matrix_ = np.linalg.inv(self.cov_) except: self.precision_matrix_ = None - self.dim = X.shape[-1] + self.dim = samples.shape[-1] self.ddof = self.dim - 1 # critical chi-square value self.confidence_level = confidence_level self.chi2_critical_ = chi2.ppf(confidence_level, df=self.ddof) + self._samples = samples + + @property + def samples(self): + return self._samples def point_estimate(self): """ @@ -234,16 +245,21 @@ class ConfidenceEllipseCLR(ConfidenceRegionABC): """ Instantiates a Confidence Ellipse in the Centered-Log Ratio (CLR) space. - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) :param confidence_level: float, the confidence level (default 0.95) """ - def __init__(self, X, confidence_level=0.95): - X = np.asarray(X) + def __init__(self, samples, confidence_level=0.95): + samples = np.asarray(samples) self.clr = CLRtransformation() - Z = self.clr(X) - self.mean_ = np.mean(X, axis=0) + Z = self.clr(samples) + self.mean_ = np.mean(samples, axis=0) self.conf_region_clr = ConfidenceEllipseSimplex(Z, confidence_level=confidence_level) + self._samples = samples + + @property + def samples(self): + return self._samples def point_estimate(self): """ @@ -273,19 +289,24 @@ class ConfidenceIntervals(ConfidenceRegionABC): """ Instantiates a region based on (independent) Confidence Intervals. - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) :param confidence_level: float, the confidence level (default 0.95) """ - def __init__(self, X, confidence_level=0.95): + def __init__(self, samples, confidence_level=0.95): assert 0 < confidence_level < 1, f'{confidence_level=} must be in range(0,1)' - X = np.asarray(X) + samples = np.asarray(samples) - self.means_ = X.mean(axis=0) + self.means_ = samples.mean(axis=0) alpha = 1-confidence_level low_perc = (alpha/2.)*100 high_perc = (1-alpha/2.)*100 - self.I_low, self.I_high = np.percentile(X, q=[low_perc, high_perc], axis=0) + self.I_low, self.I_high = np.percentile(samples, q=[low_perc, high_perc], axis=0) + self._samples = samples + + @property + def samples(self): + return self._samples def point_estimate(self): """ From 6db659e3c4eb30005042ba32e25c79839e723db3 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Tue, 18 Nov 2025 10:13:34 +0100 Subject: [PATCH 19/43] unifying n_bins in PQ and DMy --- quapy/method/confidence.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index a630fe6..c997fb3 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -624,7 +624,7 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): classifier: BaseEstimator=None, fit_classifier=True, val_split: int = 5, - n_bins: int = 4, + nbins: int = 4, fixed_bins: bool = False, num_warmup: int = 500, num_samples: int = 1_000, @@ -642,7 +642,7 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): "Run `$ pip install quapy[bayes]` to install them.") super().__init__(classifier, fit_classifier, val_split) - self.n_bins = n_bins + self.nbins = nbins self.fixed_bins = fixed_bins self.num_warmup = num_warmup self.num_samples = num_samples @@ -657,10 +657,10 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): # Compute bin limits if self.fixed_bins: # Uniform bins in [0,1] - self.bin_limits = np.linspace(0, 1, self.n_bins + 1) + self.bin_limits = np.linspace(0, 1, self.nbins + 1) else: # Quantile bins - self.bin_limits = np.quantile(y_pred, np.linspace(0, 1, self.n_bins + 1)) + self.bin_limits = np.quantile(y_pred, np.linspace(0, 1, self.nbins + 1)) # Assign each prediction to a bin bin_indices = np.digitize(y_pred, self.bin_limits[1:-1], right=True) @@ -670,14 +670,14 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): neg_mask = ~pos_mask # Count positives and negatives per bin - self.pos_hist = np.bincount(bin_indices[pos_mask], minlength=self.n_bins) - self.neg_hist = np.bincount(bin_indices[neg_mask], minlength=self.n_bins) + self.pos_hist = np.bincount(bin_indices[pos_mask], minlength=self.nbins) + self.neg_hist = np.bincount(bin_indices[neg_mask], minlength=self.nbins) def aggregate(self, classif_predictions): Px_test = classif_predictions[:, self.pos_label] test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) prevs = _bayesian.pq_stan( - self.stan_code, self.n_bins, self.pos_hist, self.neg_hist, test_hist, + self.stan_code, self.nbins, self.pos_hist, self.neg_hist, test_hist, self.num_samples, self.num_warmup, self.stan_seed ).flatten() self.prev_distribution = np.vstack([1-prevs, prevs]).T From bd59b5d190503fda7b9c7cf97c5fbb62b48cb5ee Mon Sep 17 00:00:00 2001 From: Lorenzo Volpi Date: Thu, 15 Jan 2026 18:14:50 +0100 Subject: [PATCH 20/43] fixed deprecated numpy call to in1d --- quapy/data/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index c08748f..801b968 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -758,7 +758,7 @@ def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_clas # restrict classes to only those with at least min_ipc instances classes = classes[data.counts() >= min_ipc] # filter X and y keeping only datapoints belonging to valid classes - filter_idx = np.in1d(data.y, classes) + filter_idx = np.isin(data.y, classes) X, y = data.X[filter_idx], data.y[filter_idx] # map classes to range(len(classes)) y = np.searchsorted(classes, y) From 51190c2c7d07927d899f93057aa762cecf3998ce Mon Sep 17 00:00:00 2001 From: Andrea Esuli Date: Mon, 9 Feb 2026 15:15:08 +0100 Subject: [PATCH 21/43] val_split in pcc --- quapy/method/aggregative.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index 25fc1ef..52c3720 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -402,8 +402,8 @@ class PCC(AggregativeSoftQuantifier): :param classifier: a sklearn's Estimator that generates a classifier """ - def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True): - super().__init__(classifier, fit_classifier, val_split=None) + def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True, val_split=None): + super().__init__(classifier, fit_classifier, val_split=val_split) def aggregation_fit(self, classif_predictions, labels): """ From bb423c685661c97abc00defef939abeb5d9cc68e Mon Sep 17 00:00:00 2001 From: Andrea Esuli Date: Mon, 9 Feb 2026 15:16:39 +0100 Subject: [PATCH 22/43] removed warning --- quapy/functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quapy/functional.py b/quapy/functional.py index 408c62a..f00d232 100644 --- a/quapy/functional.py +++ b/quapy/functional.py @@ -277,7 +277,7 @@ def l1_norm(prevalences: ArrayLike) -> np.ndarray: """ n_classes = prevalences.shape[-1] accum = prevalences.sum(axis=-1, keepdims=True) - prevalences = np.true_divide(prevalences, accum, where=accum > 0) + prevalences = np.true_divide(prevalences, accum, where=accum > 0, out=None) allzeros = accum.flatten() == 0 if any(allzeros): if prevalences.ndim == 1: From c1ae85950a8f432f0e31c14f630db43fc2580ee9 Mon Sep 17 00:00:00 2001 From: Andrea Esuli Date: Mon, 23 Feb 2026 09:29:45 +0100 Subject: [PATCH 23/43] Fix regex for whitespace delimiter in CSV reads --- quapy/data/datasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index 801b968..e9bc20a 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -486,7 +486,7 @@ def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize= # fall back to direct download when needed if group == "german": with download_tmp_file("statlog/german", "german.data-numeric") as tmp: - df = pd.read_csv(tmp, header=None, delim_whitespace=True) + df = pd.read_csv(tmp, header=None, sep="\\s+") X, y = df.iloc[:, 0:24].astype(float).values, df[24].astype(int).values elif group == "ctg": with download_tmp_file("00193", "CTG.xls") as tmp: @@ -500,7 +500,7 @@ def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize= y = df["NSP"].astype(int).values elif group == "semeion": with download_tmp_file("semeion", "semeion.data") as tmp: - df = pd.read_csv(tmp, header=None, sep='\s+') + df = pd.read_csv(tmp, header=None, sep="\\s+") X = df.iloc[:, 0:256].astype(float).values y = df[263].values # 263 stands for digit 8 (labels are one-hot vectors from col 256-266) else: From 1907c040adb6321119455575ea8435c4226dc251 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Tue, 3 Mar 2026 10:06:08 +0100 Subject: [PATCH 24/43] added total to lequa protocols for bag iteration --- quapy/data/_lequa.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/quapy/data/_lequa.py b/quapy/data/_lequa.py index e162f4c..7e4cf52 100644 --- a/quapy/data/_lequa.py +++ b/quapy/data/_lequa.py @@ -99,6 +99,9 @@ class SamplesFromDir(AbstractProtocol): sample, _ = self.load_fn(os.path.join(self.path_dir, f'{id}.txt')) yield sample, prevalence + def total(self): + return len(self.true_prevs) + class LabelledCollectionsFromDir(AbstractProtocol): @@ -113,6 +116,10 @@ class LabelledCollectionsFromDir(AbstractProtocol): lc = LabelledCollection.load(path=collection_path, loader_func=self.load_fn) yield lc + def total(self): + return len(self.true_prevs) + + class ResultSubmission: From e44056d860c7c8936a7f410109698010ffde0bc3 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Fri, 5 Jun 2026 14:08:06 +0200 Subject: [PATCH 25/43] integrating bayesian methods and related functionality, plus unit test refactor --- CHANGE_LOG.txt | 8 +- quapy/__init__.py | 7 +- quapy/classification/__init__.py | 4 +- quapy/classification/calibration.py | 92 +- quapy/classification/methods.py | 21 + quapy/data/datasets.py | 62 +- quapy/error.py | 79 +- quapy/functional.py | 106 +++ quapy/method/__init__.py | 8 +- quapy/method/_bayesian.py | 871 ++++++++++++++++-- quapy/method/_kdey.py | 134 ++- quapy/method/aggregative.py | 24 +- quapy/method/confidence.py | 642 ++++++++++--- quapy/protocol.py | 104 ++- quapy/tests/__init__.py | 10 + quapy/tests/_synthetic.py | 48 + ...st_datasets.py => integration_datasets.py} | 41 +- quapy/tests/integration_methods.py | 42 + quapy/tests/test_evaluation.py | 76 +- quapy/tests/test_methods.py | 129 ++- quapy/tests/test_modsel.py | 95 +- quapy/tests/test_protocols.py | 27 +- quapy/tests/test_replicability.py | 53 +- 23 files changed, 2161 insertions(+), 522 deletions(-) create mode 100644 quapy/tests/_synthetic.py rename quapy/tests/{test_datasets.py => integration_datasets.py} (75%) create mode 100644 quapy/tests/integration_methods.py diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index b6066b9..96cf26a 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -2,8 +2,15 @@ Change Log 0.2.1 ----------------- - Improved documentation of confidence regions. +- Added Bayesian KDEy and Bayesian MAPLS quantifiers. +- Added temperature calibration utilities for Bayesian confidence-aware methods. +- Added compositional CLR and ILR transformations. +- Extended KDEy with Aitchison/ILR kernels, shrinkage, and improved numerical stability. +- Added TemperatureScalingFromLogits for calibrating pretrained logits. +- Added DirichletProtocol for prevalence sampling from Dirichlet priors. - Added ReadMe method by Daniel Hopkins and Gary King - Internal index in LabelledCollection is now "lazy", and is only constructed if required. +- Improved unit testing and separated integration tests Change Log 0.2.0 ----------------- @@ -214,4 +221,3 @@ Change Log 0.1.7 any instance of BaseQuantifier), and a subclass of it called OneVsAllAggregative which implements the classify / aggregate interface. Both are instances of OneVsAll. There is a method getOneVsAll that returns the best instance based on the type of quantifier. - diff --git a/quapy/__init__.py b/quapy/__init__.py index a952fbc..182c39a 100644 --- a/quapy/__init__.py +++ b/quapy/__init__.py @@ -7,12 +7,16 @@ from . import functional from . import method from . import evaluation from . import protocol -from . import plot from . import util from . import model_selection from . import classification import os +try: + from . import plot +except ImportError: + plot = None + __version__ = '0.2.1' @@ -74,4 +78,3 @@ def _get_classifier(classifier): raise ValueError('neither classifier nor qp.environ["DEFAULT_CLS"] have been specified') return classifier - diff --git a/quapy/classification/__init__.py b/quapy/classification/__init__.py index 7aa5c77..b63e0eb 100644 --- a/quapy/classification/__init__.py +++ b/quapy/classification/__init__.py @@ -1 +1,3 @@ -from . import svmperf \ No newline at end of file +from . import calibration +from . import methods +from . import svmperf diff --git a/quapy/classification/calibration.py b/quapy/classification/calibration.py index 0f5e9f7..04ce2f9 100644 --- a/quapy/classification/calibration.py +++ b/quapy/classification/calibration.py @@ -1,8 +1,9 @@ from copy import deepcopy -from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling from sklearn.base import BaseEstimator, clone from sklearn.model_selection import cross_val_predict, train_test_split +from sklearn.preprocessing import LabelEncoder +from sklearn.utils.validation import check_X_y import numpy as np @@ -11,6 +12,17 @@ import numpy as np # see https://github.com/kundajelab/abstention +def _require_abstention_calibration(): + try: + from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling + except ImportError as exc: + raise ImportError( + "Calibration methods in quapy.classification.calibration require the optional " + "'abstention' package." + ) from exc + return NoBiasVectorScaling, TempScaling, VectorScaling + + class RecalibratedProbabilisticClassifier: """ Abstract class for (re)calibration method from `abstention.calibration`, as defined in @@ -142,6 +154,7 @@ class NBVSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + NoBiasVectorScaling, _, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = NoBiasVectorScaling(verbose=verbose) self.val_split = val_split @@ -164,6 +177,7 @@ class BCTSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, TempScaling, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = TempScaling(verbose=verbose, bias_positions='all') self.val_split = val_split @@ -186,6 +200,7 @@ class TSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, TempScaling, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = TempScaling(verbose=verbose) self.val_split = val_split @@ -208,9 +223,84 @@ class VSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, _, VectorScaling = _require_abstention_calibration() self.classifier = classifier self.calibrator = VectorScaling(verbose=verbose) self.val_split = val_split self.n_jobs = n_jobs self.verbose = verbose + +class TemperatureScalingFromLogits(BaseEstimator): + """ + Calibrates a matrix of logits by learning a temperature-scaling mapping + with the calibration methods from `abstention.calibration`. + + This estimator is useful when the inputs are already logits produced by a + pretrained classifier, and the goal is to transform them directly into + calibrated posterior probabilities without retraining the underlying model. + + :param bias_corrected: if True, uses Bias-Corrected Temperature Scaling + (BCTS); otherwise, uses standard Temperature Scaling (TS) + :param verbose: whether the underlying calibrator should display progress + information + """ + + def __init__(self, bias_corrected=False, verbose=False): + self.bias_corrected = bias_corrected + self.verbose = verbose + + def fit(self, X, y): + """ + Fits the logits calibrator. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :param y: array-like of shape `(n_samples,)` containing class labels + :return: self + """ + X, y = check_X_y(X, y) + + self.label_encoder_ = LabelEncoder() + y_enc = self.label_encoder_.fit_transform(y) + self.classes_ = self.label_encoder_.classes_ + + n_classes = len(self.classes_) + logits_dim = X.shape[1] + if n_classes != logits_dim: + raise ValueError( + f'mismatch between the number of classes ({n_classes}) and the ' + f'dimensionality of the logits ({logits_dim})' + ) + + _, TempScaling, _ = _require_abstention_calibration() + calibrator = TempScaling( + verbose=self.verbose, + bias_positions='all' if self.bias_corrected else [], + ) + self.calibrator_ = calibrator + self.calibration_function_ = calibrator(X, np.eye(n_classes)[y_enc]) + return self + + def predict_proba(self, X): + """ + Converts logits into calibrated posterior probabilities. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :return: array-like of shape `(n_samples, n_classes)` with calibrated + posterior probabilities + """ + return self.calibration_function_(X) + + def predict(self, X): + """ + Predicts class labels after calibration. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :return: array-like of shape `(n_samples,)` with class label + predictions + """ + posteriors = self.predict_proba(X) + return self.label_encoder_.inverse_transform(np.argmax(posteriors, axis=1)) diff --git a/quapy/classification/methods.py b/quapy/classification/methods.py index 85300d0..1876c4a 100644 --- a/quapy/classification/methods.py +++ b/quapy/classification/methods.py @@ -1,3 +1,4 @@ +import numpy as np from sklearn.base import BaseEstimator from sklearn.decomposition import TruncatedSVD from sklearn.linear_model import LogisticRegression @@ -95,3 +96,23 @@ class LowRankLogisticRegression(BaseEstimator): if self.pca is None: return X return self.pca.transform(X) + + +class MockClassifierFromPosteriors(BaseEstimator): + """ + Mock classifier that bypasses classifier training when the input instances + are already posterior probabilities produced by a pretrained probabilistic + classifier. + + :param X: arrays of shape `(n_samples, n_classes)` are interpreted as posterior probabilities + """ + + def fit(self, X, y): + self.classes_ = np.sort(np.unique(y)) + return self + + def predict(self, X): + return np.argmax(X, axis=1) + + def predict_proba(self, X): + return X diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index e9bc20a..8c235e3 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -3,7 +3,6 @@ from contextlib import contextmanager import zipfile from os.path import join import pandas as pd -from ucimlrepo import fetch_ucirepo from quapy.data.base import Dataset, LabelledCollection from quapy.data.preprocessing import text2tfidf, reduce_columns from quapy.data.preprocessing import standardize as standardizer @@ -12,6 +11,17 @@ from quapy.util import download_file_if_not_exists, download_file, get_quapy_hom from sklearn.preprocessing import StandardScaler +def _fetch_ucirepo(*args, **kwargs): + try: + from ucimlrepo import fetch_ucirepo + except ImportError as exc: + raise ImportError( + "UCI dataset fetching requires the optional 'ucimlrepo' package. " + "Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset." + ) from exc + return fetch_ucirepo(*args, **kwargs) + + REVIEWS_SENTIMENT_DATASETS = ['hp', 'kindle', 'imdb'] TWITTER_SENTIMENT_DATASETS_TEST = [ @@ -486,7 +496,7 @@ def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize= # fall back to direct download when needed if group == "german": with download_tmp_file("statlog/german", "german.data-numeric") as tmp: - df = pd.read_csv(tmp, header=None, sep="\\s+") + df = pd.read_csv(tmp, header=None, delim_whitespace=True) X, y = df.iloc[:, 0:24].astype(float).values, df[24].astype(int).values elif group == "ctg": with download_tmp_file("00193", "CTG.xls") as tmp: @@ -500,11 +510,11 @@ def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize= y = df["NSP"].astype(int).values elif group == "semeion": with download_tmp_file("semeion", "semeion.data") as tmp: - df = pd.read_csv(tmp, header=None, sep="\\s+") + df = pd.read_csv(tmp, header=None, sep='\\s+') X = df.iloc[:, 0:256].astype(float).values y = df[263].values # 263 stands for digit 8 (labels are one-hot vectors from col 256-266) else: - df = fetch_ucirepo(id=id) + df = _fetch_ucirepo(id=id) X, y = df.data.features.to_numpy(), df.data.targets.to_numpy().squeeze() # transform data when needed before returning (returned data will be pickled) @@ -616,8 +626,8 @@ def fetch_UCIMulticlassDataset( are taken for training, and the rest (irrespective of `min_test_split`) is taken for test. :param max_train_instances: maximum number of instances to keep for training (defaults to 25000); set to -1 or None to avoid this check - :param min_class_support: minimum number of istances per class. Classes with fewer instances - are discarded (deafult is 100) + :param min_class_support: integer or float, the minimum number or proportion of istances per class. + Classes with fewer instances are discarded (deafult is 100). :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested, standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the training portion of the data. @@ -673,6 +683,11 @@ def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_clas f'Name {dataset_name} does not match any known dataset from the ' \ f'UCI Machine Learning datasets repository (multiclass). ' \ f'Valid ones are {UCI_MULTICLASS_DATASETS}' + + assert (min_class_support is None or + ((isinstance(min_class_support, int) and min_class_support >= 0) or + (isinstance(min_class_support, float) and 0. <= min_class_support < 1.))), \ + f'invalid value for {min_class_support=}; expected non negative integer or float in [0,1)' if data_home is None: data_home = get_quapy_home() @@ -739,24 +754,41 @@ def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_clas file = join(data_home, 'uci_multiclass', dataset_name+'.pkl') + def dummify_categorical_features(df_features, dataset_id): + categorical_features = { + 158: ["S1", "C1", "S2", "C2", "S3", "C3", "S4", "C4", "S5", "C5"], # poker_hand + } + + categorical = categorical_features.get(dataset_id, []) + + X = df_features.copy() + if categorical: + X[categorical] = X[categorical].astype("category") + X = pd.get_dummies(X, columns=categorical, drop_first=True) + + return X + def download(id, name): - df = fetch_ucirepo(id=id) + df = _fetch_ucirepo(id=id) - df.data.features = pd.get_dummies(df.data.features, drop_first=True) - X, y = df.data.features.to_numpy(dtype=np.float64), df.data.targets.to_numpy().squeeze() + X_df = dummify_categorical_features(df.data.features, id) + X = X_df.to_numpy(dtype=np.float64) + y = df.data.targets.to_numpy().squeeze() - assert y.ndim == 1, 'more than one y' + assert y.ndim == 1, f'error: the dataset {id=} {name=} has more than one target variable' classes = np.sort(np.unique(y)) y = np.searchsorted(classes, y) return LabelledCollection(X, y) - def filter_classes(data: LabelledCollection, min_ipc): - if min_ipc is None: - min_ipc = 0 + def filter_classes(data: LabelledCollection, min_class_support): + if min_class_support is None or min_class_support == 0.: + return data + if isinstance(min_class_support, float): + min_class_support = int(len(data) * min_class_support) classes = data.classes_ - # restrict classes to only those with at least min_ipc instances - classes = classes[data.counts() >= min_ipc] + # restrict classes to only those with at least min_class_support instances + classes = classes[data.counts() >= min_class_support] # filter X and y keeping only datapoints belonging to valid classes filter_idx = np.isin(data.y, classes) X, y = data.X[filter_idx], data.y[filter_idx] diff --git a/quapy/error.py b/quapy/error.py index eb42cd6..621d08d 100644 --- a/quapy/error.py +++ b/quapy/error.py @@ -128,6 +128,78 @@ def se(prevs_true, prevs_hat): return ((prevs_hat - prevs_true) ** 2).mean(axis=-1) +def sre(prevs_true, prevs_hat, prevs_train, eps=0.): + """ + Computes the squared ratio error between two prevalence vectors. + The squared ratio error between prevalence vectors :math:`p` and + :math:`\\hat{p}` with training prevalence :math:`p^{tr}` is: + :math:`SRE(p,\\hat{p},p^{tr})=\\frac{1}{|\\mathcal{Y}|}\\sum_{i \\in \\mathcal{Y}}(w_i-\\hat{w}_i)^2`, + where :math:`w_i=\\frac{p_i}{p^{tr}_i}`. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :param prevs_train: array-like with the training prevalence values, or a single + prevalence vector when all comparisons refer to the same training set + :param eps: smoothing factor for the prevalence values (default 0, i.e., no smoothing) + :return: squared ratio error + """ + prevs_true = np.asarray(prevs_true) + prevs_hat = np.asarray(prevs_hat) + prevs_train = np.asarray(prevs_train) + assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape=} vs {prevs_hat.shape=}' + assert prevs_true.shape[-1] == prevs_train.shape[-1], 'wrong shape for training prevalence' + if prevs_true.ndim == 2 and prevs_train.ndim == 1: + prevs_train = np.tile(prevs_train, reps=(prevs_true.shape[0], 1)) + if eps > 0: + prevs_true = smooth(prevs_true, eps) + prevs_hat = smooth(prevs_hat, eps) + prevs_train = smooth(prevs_train, eps) + + n_classes = prevs_true.shape[-1] + w = prevs_true / prevs_train + w_hat = prevs_hat / prevs_train + return (1. / n_classes) * np.sum((w - w_hat) ** 2., axis=-1) + + +def msre(prevs_true, prevs_hat, prevs_train, eps=0.): + """ + Computes the mean squared ratio error (see :meth:`quapy.error.sre`) across the sample pairs. + + :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values + :param prevs_hat: array-like of shape equal to prevs_true with the predicted prevalence values + :param prevs_train: array-like with the training prevalence values + :param eps: smoothing factor (default 0, i.e., no smoothing) + :return: mean squared ratio error + """ + return np.mean(sre(prevs_true, prevs_hat, prevs_train, eps)) + + +def aitchisondist(prevs_true, prevs_hat): + """ + Computes the Aitchison distance between two prevalence vectors. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :return: Aitchison distance + """ + from quapy.functional import CLRtransformation + + clr = CLRtransformation() + return np.linalg.norm(clr(prevs_true) - clr(prevs_hat), axis=-1) + + +def maitchisondist(prevs_true, prevs_hat): + """ + Computes the mean Aitchison distance (see :meth:`quapy.error.aitchisondist`) + across the sample pairs. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :return: mean Aitchison distance + """ + return np.mean(aitchisondist(prevs_true, prevs_hat)) + + def mkld(prevs_true, prevs_hat, eps=None): """Computes the mean Kullback-Leibler divergence (see :meth:`quapy.error.kld`) across the sample pairs. The distributions are smoothed using the `eps` factor @@ -374,8 +446,8 @@ def __check_eps(eps=None): CLASSIFICATION_ERROR = {f1e, acce} -QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld} -QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld} +QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld, msre, maitchisondist} +QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld, sre, aitchisondist} QUANTIFICATION_ERROR_SMOOTH = {kld, nkld, rae, nrae, mkld, mnkld, mrae} CLASSIFICATION_ERROR_NAMES = {func.__name__ for func in CLASSIFICATION_ERROR} QUANTIFICATION_ERROR_NAMES = {func.__name__ for func in QUANTIFICATION_ERROR} @@ -387,6 +459,9 @@ ERROR_NAMES = \ f1_error = f1e acc_error = acce mean_absolute_error = mae +squared_ratio_error = sre +dist_aitchison = aitchisondist +mean_dist_aitchison = maitchisondist absolute_error = ae mean_relative_absolute_error = mrae relative_absolute_error = rae diff --git a/quapy/functional.py b/quapy/functional.py index f00d232..bd62377 100644 --- a/quapy/functional.py +++ b/quapy/functional.py @@ -1,11 +1,15 @@ import warnings +from abc import ABC, abstractmethod from collections import defaultdict +from functools import lru_cache from typing import Literal, Union, Callable from numpy.typing import ArrayLike import scipy import numpy as np +import quapy as qp + # ------------------------------------------------------------------------------------------ # General utils @@ -649,3 +653,105 @@ def solve_adjustment( raise ValueError(f'unknown {solver=}') +# ------------------------------------------------------------------------------------------ +# Transformations from Compositional analysis +# ------------------------------------------------------------------------------------------ + +class CompositionalTransformation(ABC): + """ + Abstract class of transformations for compositional data. + """ + + EPSILON = 1e-12 + + @abstractmethod + def __call__(self, X): + ... + + @abstractmethod + def inverse(self, Z): + ... + + +class CLRtransformation(CompositionalTransformation): + """ + Centered log-ratio (CLR) transformation. + """ + + def __call__(self, X): + X = np.asarray(X) + X = qp.error.smooth(X, self.EPSILON) + geometric_mean = np.exp(np.mean(np.log(X), axis=-1, keepdims=True)) + return np.log(X / geometric_mean) + + def inverse(self, Z): + return scipy.special.softmax(Z, axis=-1) + + +class ILRtransformation(CompositionalTransformation): + """ + Isometric log-ratio (ILR) transformation. + """ + + def __call__(self, X): + X = np.asarray(X) + X = qp.error.smooth(X, self.EPSILON) + basis = self.get_V(X.shape[-1]) + return np.log(X) @ basis.T + + def inverse(self, Z): + Z = np.asarray(Z) + basis = self.get_V(Z.shape[-1] + 1) + logp = Z @ basis + p = np.exp(logp) + return p / np.sum(p, axis=-1, keepdims=True) + + @lru_cache(maxsize=None) + def get_V(self, k): + helmert = np.zeros((k, k)) + for i in range(1, k): + helmert[i, :i] = 1 + helmert[i, i] = -i + helmert[i] = helmert[i] / np.sqrt(i * (i + 1)) + return helmert[1:, :] + + +def normalized_entropy(p): + """ + Computes the normalized Shannon entropy of a prevalence vector. + + :param p: array-like prevalence vector summing to 1 + :return: float in [0,1] + """ + p = np.asarray(p) + entropy = scipy.stats.entropy(p) + max_entropy = np.log(len(p)) + return np.clip(entropy / max_entropy, 0, 1) + + +def antagonistic_prevalence(p, strength=1): + """ + Reflects a prevalence vector in ILR space and maps it back to the simplex. + + :param p: array-like prevalence vector + :param strength: reflection strength in ILR space + :return: prevalence vector in the simplex + """ + ilr = ILRtransformation() + z = ilr(p) + z_ant = -strength * z + return ilr.inverse(z_ant) + + +def in_simplex(x, atol=1e-8): + """ + Checks whether points lie in the probability simplex. + + :param x: array-like of shape `(n_classes,)` or `(n_points, n_classes)` + :param atol: numerical tolerance for the unit-sum check + :return: boolean or boolean array + """ + x = np.asarray(x) + non_negative = np.all(x >= 0, axis=-1) + sum_to_one = np.isclose(x.sum(axis=-1), 1.0, atol=atol) + return non_negative & sum_to_one diff --git a/quapy/method/__init__.py b/quapy/method/__init__.py index ab7a59b..a83c42b 100644 --- a/quapy/method/__init__.py +++ b/quapy/method/__init__.py @@ -3,6 +3,7 @@ from sklearn.exceptions import ConvergenceWarning warnings.simplefilter("ignore", ConvergenceWarning) from . import confidence +from . import _bayesian from . import base from . import aggregative from . import non_aggregative @@ -29,6 +30,8 @@ AGGREGATIVE_METHODS = { aggregative.KDEyHD, # aggregative.OneVsAllAggregative, confidence.BayesianCC, + _bayesian.BayesianKDEy, + _bayesian.BayesianMAPLS, confidence.PQ, } @@ -53,7 +56,9 @@ MULTICLASS_METHODS = { aggregative.KDEyML, aggregative.KDEyCS, aggregative.KDEyHD, - confidence.BayesianCC + confidence.BayesianCC, + _bayesian.BayesianKDEy, + _bayesian.BayesianMAPLS, } NON_AGGREGATIVE_METHODS = { @@ -71,4 +76,3 @@ QUANTIFICATION_METHODS = AGGREGATIVE_METHODS | NON_AGGREGATIVE_METHODS | META_ME - diff --git a/quapy/method/_bayesian.py b/quapy/method/_bayesian.py index da65eed..90a10b6 100644 --- a/quapy/method/_bayesian.py +++ b/quapy/method/_bayesian.py @@ -1,22 +1,49 @@ """ -Utility functions for `Bayesian quantification `_ methods. +Utilities and methods for Bayesian quantification. """ -import numpy as np +import contextlib +import copy import importlib.resources +import logging +import os +import sys +from collections.abc import Iterable +from numbers import Number, Real + +import numpy as np +from joblib import Parallel, delayed +from sklearn.base import BaseEstimator +from tqdm import tqdm + +import quapy as qp +import quapy.functional as F +from quapy.data import LabelledCollection +from quapy.method._kdey import KDEBase +from quapy.method.aggregative import AggregativeSoftQuantifier +from quapy.method.confidence import ConfidenceRegionABC, WithConfidenceABC +from quapy.protocol import AbstractProtocol try: import jax import jax.numpy as jnp + import jax.random as jrandom + from jax.scipy.special import logsumexp as jax_logsumexp import numpyro import numpyro.distributions as dist + from numpyro.infer import MCMC, NUTS import stan + import stan.common DEPENDENCIES_INSTALLED = True except ImportError: jax = None jnp = None + jrandom = None + jax_logsumexp = None numpyro = None dist = None + MCMC = None + NUTS = None stan = None DEPENDENCIES_INSTALLED = False @@ -27,28 +54,124 @@ P_TEST_C: str = "P_test(C)" P_C_COND_Y: str = "P(C|Y)" -def model(n_c_unlabeled: np.ndarray, n_y_and_c_labeled: np.ndarray) -> None: - """ - Defines a probabilistic model in `NumPyro `_. +def _require_bayesian_dependencies(): + if not DEPENDENCIES_INSTALLED: + raise ImportError( + "Auxiliary dependencies are required. " + "Run `$ pip install quapy[bayes]` to install them." + ) - :param n_c_unlabeled: a `np.ndarray` of shape `(n_predicted_classes,)` - with entry `c` being the number of instances predicted as class `c`. - :param n_y_and_c_labeled: a `np.ndarray` of shape `(n_classes, n_predicted_classes)` - with entry `(y, c)` being the number of instances labeled as class `y` and predicted as class `c`. + +def _resolve_dirichlet_prior(prior, n_classes, *, allow_mapls_priors=False, n_test=None, map_prev=None, map_lambda=None): + if isinstance(prior, str): + if prior == 'uniform': + return np.ones(n_classes, dtype=float) + if allow_mapls_priors and prior in {'map', 'map2'}: + if n_test is None or map_prev is None: + raise ValueError('MAPLS priors require n_test and map_prev') + if prior == 'map': + lam = map_lambda + else: + lam = get_lambda( + test_probs=map_prev["test_probs"], + pz=map_prev["train_prev"], + q_prior=map_prev["map_estimate"], + dvg=kl_div, + ) + alpha_0 = alpha0_from_lambda(lam, n_test=n_test, n_classes=n_classes) + return np.full(n_classes, alpha_0, dtype=float) + raise ValueError(f"unknown prior specification {prior!r}") + if isinstance(prior, Number): + return np.full(n_classes, float(prior), dtype=float) + + alpha = np.asarray(prior, dtype=float) + if alpha.ndim != 1 or len(alpha) != n_classes: + raise ValueError(f'wrong shape for prior; expected {n_classes} values, found shape {alpha.shape}') + return alpha + + +def _validate_temperature(temperature): + if not isinstance(temperature, Real) or temperature <= 0: + raise ValueError(f'expected a positive real value for temperature; found {temperature!r}') + return float(temperature) + + +def model_bayesianCC( + n_c_unlabeled: np.ndarray, + n_y_and_c_labeled: np.ndarray, + temperature: float, + alpha: np.ndarray, +) -> None: + """ + NumPyro model for BayesianCC. """ n_y_labeled = n_y_and_c_labeled.sum(axis=1) - K = len(n_c_unlabeled) - L = len(n_y_labeled) + n_pred_classes = len(n_c_unlabeled) + n_classes = len(n_y_labeled) - pi_ = numpyro.sample(P_TEST_Y, dist.Dirichlet(jnp.ones(L))) - p_c_cond_y = numpyro.sample(P_C_COND_Y, dist.Dirichlet(jnp.ones(K).repeat(L).reshape(L, K))) + pi_ = numpyro.sample(P_TEST_Y, dist.Dirichlet(jnp.asarray(alpha, dtype=jnp.float32))) + p_c_cond_y = numpyro.sample( + P_C_COND_Y, + dist.Dirichlet(jnp.ones(n_pred_classes).repeat(n_classes).reshape(n_classes, n_pred_classes)), + ) - with numpyro.plate('plate', L): - numpyro.sample('F_yc', dist.Multinomial(n_y_labeled, p_c_cond_y), obs=n_y_and_c_labeled) + if temperature == 1.0: + with numpyro.plate('plate', n_classes): + numpyro.sample('F_yc', dist.Multinomial(n_y_labeled, p_c_cond_y), obs=n_y_and_c_labeled) + + p_c = numpyro.deterministic(P_TEST_C, jnp.einsum("yc,y->c", p_c_cond_y, pi_)) + numpyro.sample('N_c', dist.Multinomial(jnp.sum(n_c_unlabeled), p_c), obs=n_c_unlabeled) + return + + with numpyro.plate('plate_y', n_classes): + logp_f = dist.Multinomial(n_y_labeled, p_c_cond_y).log_prob(n_y_and_c_labeled) + + numpyro.factor('F_yc_loglik', jnp.sum(logp_f) / temperature) p_c = numpyro.deterministic(P_TEST_C, jnp.einsum("yc,y->c", p_c_cond_y, pi_)) - numpyro.sample('N_c', dist.Multinomial(jnp.sum(n_c_unlabeled), p_c), obs=n_c_unlabeled) + logp_n = dist.Multinomial(jnp.sum(n_c_unlabeled), p_c).log_prob(n_c_unlabeled) + numpyro.factor('N_c_loglik', logp_n / temperature) + + +def model(n_c_unlabeled: np.ndarray, n_y_and_c_labeled: np.ndarray) -> None: + """ + Backward-compatible BayesianCC model with a uniform prior. + """ + alpha = np.ones(n_y_and_c_labeled.shape[0], dtype=float) + return model_bayesianCC(n_c_unlabeled, n_y_and_c_labeled, temperature=1.0, alpha=alpha) + + +def sample_posterior_bayesianCC( + n_c_unlabeled: np.ndarray, + n_y_and_c_labeled: np.ndarray, + num_warmup: int, + num_samples: int, + alpha: np.ndarray, + temperature: float = 1.0, + seed: int = 0, +) -> dict: + """ + Samples from the BayesianCC posterior using NumPyro. + """ + _require_bayesian_dependencies() + temperature = _validate_temperature(temperature) + + mcmc = numpyro.infer.MCMC( + numpyro.infer.NUTS(model_bayesianCC), + num_warmup=num_warmup, + num_samples=num_samples, + progress_bar=False, + ) + rng_key = jax.random.PRNGKey(seed) + mcmc.run( + rng_key, + n_c_unlabeled=n_c_unlabeled, + n_y_and_c_labeled=n_y_and_c_labeled, + temperature=temperature, + alpha=alpha, + ) + return mcmc.get_samples() def sample_posterior( @@ -59,77 +182,679 @@ def sample_posterior( seed: int = 0, ) -> dict: """ - Samples from the Bayesian quantification model in NumPyro using the - `NUTS `_ sampler. - - :param n_c_unlabeled: a `np.ndarray` of shape `(n_predicted_classes,)` - with entry `c` being the number of instances predicted as class `c`. - :param n_y_and_c_labeled: a `np.ndarray` of shape `(n_classes, n_predicted_classes)` - with entry `(y, c)` being the number of instances labeled as class `y` and predicted as class `c`. - :param num_warmup: the number of warmup steps. - :param num_samples: the number of samples to draw. - :seed: the random seed. - :return: a `dict` with the samples. The keys are the names of the latent variables. + Backward-compatible wrapper around BayesianCC sampling. """ - mcmc = numpyro.infer.MCMC( - numpyro.infer.NUTS(model), + alpha = np.ones(n_y_and_c_labeled.shape[0], dtype=float) + return sample_posterior_bayesianCC( + n_c_unlabeled=n_c_unlabeled, + n_y_and_c_labeled=n_y_and_c_labeled, num_warmup=num_warmup, num_samples=num_samples, - progress_bar=False + alpha=alpha, + temperature=1.0, + seed=seed, ) - rng_key = jax.random.PRNGKey(seed) - mcmc.run(rng_key, n_c_unlabeled=n_c_unlabeled, n_y_and_c_labeled=n_y_and_c_labeled) - return mcmc.get_samples() - def load_stan_file(): return importlib.resources.files('quapy.method').joinpath('stan/pq.stan').read_text(encoding='utf-8') + +@contextlib.contextmanager +def _suppress_stan_logging(): + with open(os.devnull, "w") as devnull: + old_stderr = sys.stderr + sys.stderr = devnull + try: + yield + finally: + sys.stderr = old_stderr + + def pq_stan(stan_code, n_bins, pos_hist, neg_hist, test_hist, number_of_samples, num_warmup, stan_seed): """ - Perform Bayesian prevalence estimation using a Stan model for probabilistic quantification. - - This function builds and samples from a Stan model that implements a bin-based Bayesian - quantifier. It uses the class-conditional histograms of the classifier - outputs for positive and negative examples, along with the test histogram, to estimate - the posterior distribution of prevalence in the test set. - - Parameters - ---------- - stan_code : str - The Stan model code as a string. - n_bins : int - Number of bins used to build the histograms for positive and negative examples. - pos_hist : array-like of shape (n_bins,) - Histogram counts of the classifier outputs for the positive class. - neg_hist : array-like of shape (n_bins,) - Histogram counts of the classifier outputs for the negative class. - test_hist : array-like of shape (n_bins,) - Histogram counts of the classifier outputs for the test set, binned using the same bins. - number_of_samples : int - Number of post-warmup samples to draw from the Stan posterior. - num_warmup : int - Number of warmup iterations for the sampler. - stan_seed : int - Random seed for Stan model compilation and sampling, ensuring reproducibility. - - Returns - ------- - prev_samples : numpy.ndarray - An array of posterior samples of the prevalence (`prev`) in the test set. - Each element corresponds to one draw from the posterior distribution. + Samples posterior prevalences for PQ from a Stan model. """ + _require_bayesian_dependencies() + logging.getLogger("stan.common").setLevel(logging.ERROR) stan_data = { - 'n_bucket': n_bins, - 'train_neg': neg_hist.tolist(), - 'train_pos': pos_hist.tolist(), - 'test': test_hist.tolist(), - 'posterior': 1 - } + 'n_bucket': n_bins, + 'train_neg': neg_hist.tolist(), + 'train_pos': pos_hist.tolist(), + 'test': test_hist.tolist(), + 'posterior': 1, + } - stan_model = stan.build(stan_code, data=stan_data, random_seed=stan_seed) - fit = stan_model.sample(num_chains=1, num_samples=number_of_samples,num_warmup=num_warmup) + with _suppress_stan_logging(): + stan_model = stan.build(stan_code, data=stan_data, random_seed=stan_seed) + fit = stan_model.sample(num_chains=1, num_samples=number_of_samples, num_warmup=num_warmup) return fit['prev'] + + +class BayesianKDEy(AggregativeSoftQuantifier, KDEBase, WithConfidenceABC): + """ + Bayesian version of KDEy. + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case + the classifier is taken to be the one indicated in + `qp.environ['DEFAULT_CLS']` + :param fit_classifier: whether to train the classifier, or consider it + already fit + :param val_split: specifies the data used for generating classifier + predictions. This specification can be made as float in (0, 1) + indicating the proportion of stratified held-out validation set to be + extracted from the training set; or as an integer (default 5), + indicating that the predictions are to be generated in a `k`-fold + cross-validation manner (with this integer indicating the value for + `k`); or as a tuple `(X,y)` defining the specific set of data to use + for validation. Set to None when the method does not require any + validation data, in order to avoid that some portion of the training + data be wasted. + :param kernel: kernel function for KDE. Available kernels include + {'gaussian', 'aitchison', 'ilr'} (default 'gaussian') + :param bandwidth: bandwidth for the kernel (default 0.1) + :param shrinkage: regularization strength for Aitchison/ILR kernels + (default 0.0) + :param num_warmup: number of warmup iterations for the MCMC sampler + (default 500) + :param num_samples: number of samples to draw from the posterior + (default 1000) + :param mcmc_seed: random seed for the MCMC sampler (default 0) + :param confidence_level: float in [0,1] to construct a confidence region + around the point estimate (default 0.95) + :param region: string, set to `intervals` for constructing confidence + intervals (default), or to `ellipse` for constructing an ellipse in + the probability simplex, or to `ellipse-clr` for constructing an + ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param temperature: temperature (>0) for posterior calibration + (default 1.) + :param prior: an array-like with the alpha parameters of a Dirichlet + prior, a scalar real value to be broadcast to all classes, or the + string 'uniform' for a uniform, uninformative prior (default) + :param verbose: bool, whether to display the progress bar + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier=True, + val_split: int = 5, + kernel='gaussian', + bandwidth=0.1, + shrinkage=0.0, + num_warmup: int = 500, + num_samples: int = 1_000, + mcmc_seed: int = 0, + confidence_level: float = 0.95, + region: str = 'intervals', + temperature: float = 1.0, + prior='uniform', + verbose: bool = False, + ): + _require_bayesian_dependencies() + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + + self.kernel = KDEBase._check_kernel(kernel) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, self.kernel) + assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)' + assert self.kernel != 'gaussian' or shrinkage == 0, \ + 'shrinkage is only supported for Aitchison/ILR kernels' + + super().__init__(classifier, fit_classifier, val_split) + self.shrinkage = float(shrinkage) + self.num_warmup = num_warmup + self.num_samples = num_samples + self.mcmc_seed = mcmc_seed + self.confidence_level = confidence_level + self.region = region + self.temperature = _validate_temperature(temperature) + self.prior = prior + self.verbose = verbose + self.prevalence_samples = None + + def aggregation_fit(self, classif_predictions, labels): + self.mix_densities = self.get_mixture_components( + classif_predictions, + labels, + self.classes_, + self.bandwidth, + self.kernel, + ) + return self + + def sample_from_posterior(self, classif_predictions): + test_log_densities = np.asarray( + [self.pdf(kde_i, classif_predictions, self.kernel, log_densities=True) for kde_i in self.mix_densities] + ) + alpha = _resolve_dirichlet_prior(self.prior, len(self.mix_densities)) + + mcmc = MCMC( + NUTS(self._numpyro_model), + num_warmup=self.num_warmup, + num_samples=self.num_samples, + num_chains=1, + progress_bar=self.verbose, + ) + mcmc.run(jrandom.PRNGKey(self.mcmc_seed), test_log_densities=test_log_densities, alpha=alpha) + self.prevalence_samples = np.asarray(mcmc.get_samples()["prev"]) + return self.prevalence_samples + + def aggregate(self, classif_predictions: np.ndarray): + return self.sample_from_posterior(classif_predictions).mean(axis=0) + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + confidence_level = self.confidence_level if confidence_level is None else confidence_level + classif_predictions = self.classify(instances) + point_estimate = self.aggregate(classif_predictions) + region = WithConfidenceABC.construct_region( + self.prevalence_samples, + confidence_level=confidence_level, + method=self.region, + ) + return point_estimate, region + + def _numpyro_model(self, test_log_densities, alpha): + prev = numpyro.sample("prev", dist.Dirichlet(jnp.asarray(alpha))) + log_likelihood = jnp.sum(jax_logsumexp(jnp.log(prev)[:, None] + test_log_densities, axis=0)) + numpyro.factor("loglik", (1.0 / self.temperature) * log_likelihood) + + +class _JaxILRTransformation(F.CompositionalTransformation): + """ + JAX-backed ILR transform used inside Bayesian MAPLS. + """ + + def __call__(self, X): + X = jnp.asarray(X) + X = qp.error.smooth(np.asarray(X), self.EPSILON) + X = jnp.asarray(X) + basis = jnp.asarray(self.get_V(X.shape[-1])) + return jnp.log(X) @ basis.T + + def inverse(self, Z): + Z = jnp.asarray(Z) + basis = jnp.asarray(self.get_V(Z.shape[-1] + 1)) + logp = Z @ basis + p = jnp.exp(logp) + return p / jnp.sum(p, axis=-1, keepdims=True) + + def get_V(self, k): + return F.ILRtransformation().get_V(k) + + +class BayesianMAPLS(AggregativeSoftQuantifier, WithConfidenceABC): + """ + Bayesian variant of the MLLS/EMQ method proposed by + Ye, Changkun, et al. "Label shift estimation for class-imbalance problem: + A bayesian approach." Proceedings of the IEEE/CVF Winter Conference on + Applications of Computer Vision. 2024. + + Code adapted from: + https://github.com/ChangkunYe/MAPLS/blob/main/label_shift/mapls.py + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case + the classifier is taken to be the one indicated in + `qp.environ['DEFAULT_CLS']` + :param fit_classifier: whether to train the classifier, or consider it + already fit + :param val_split: specifies the data used for generating classifier + predictions. This specification can be made as float in (0, 1) + indicating the proportion of stratified held-out validation set to be + extracted from the training set; or as an integer (default 5), + indicating that the predictions are to be generated in a `k`-fold + cross-validation manner (with this integer indicating the value for + `k`); or as a tuple `(X,y)` defining the specific set of data to use + for validation. Set to None when the method does not require any + validation data, in order to avoid that some portion of the training + data be wasted. + :param exact_train_prev: set to True (default) for using the true training + prevalence as the initial observation; set to False for computing the + training prevalence as an estimate of it, i.e., as the expected value + of the posterior probabilities of the training instances. + :param num_warmup: number of warmup iterations for the MCMC sampler + (default 500) + :param num_samples: number of samples to draw from the posterior + (default 1000) + :param mcmc_seed: random seed for the MCMC sampler (default 0) + :param confidence_level: float in [0,1] to construct a confidence region + around the point estimate (default 0.95) + :param region: string, set to `intervals` for constructing confidence + intervals (default), or to `ellipse` for constructing an ellipse in + the probability simplex, or to `ellipse-clr` for constructing an + ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param temperature: temperature (>0) for posterior calibration + (default 1.) + :param prior: an array-like with the alpha parameters of a Dirichlet + prior, a scalar real value to be broadcast to all classes, or one of + {'uniform', 'map', 'map2'} (default 'uniform') + :param mapls_chain_init: whether to initialize the Markov chain with a + preliminary EM point estimate (default True) + :param verbose: bool, whether to display the progress bar + (default False) + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier=True, + val_split: int = 5, + exact_train_prev=True, + num_warmup: int = 500, + num_samples: int = 1_000, + mcmc_seed: int = 0, + confidence_level: float = 0.95, + region: str = 'intervals', + temperature: float = 1.0, + prior='uniform', + mapls_chain_init=True, + verbose=False, + ): + _require_bayesian_dependencies() + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + if not ( + (isinstance(prior, str) and prior in {'uniform', 'map', 'map2'}) + or isinstance(prior, Number) + or (isinstance(prior, Iterable) and all(isinstance(v, Number) for v in prior)) + ): + raise ValueError( + f'wrong type for {prior=}; expected one of {{"uniform", "map", "map2"}}, ' + 'a real scalar, or an array-like of real values' + ) + + super().__init__(classifier, fit_classifier, val_split) + self.exact_train_prev = exact_train_prev + self.num_warmup = num_warmup + self.num_samples = num_samples + self.mcmc_seed = mcmc_seed + self.confidence_level = confidence_level + self.region = region + self.temperature = _validate_temperature(temperature) + self.prior = prior + self.mapls_chain_init = mapls_chain_init + self.verbose = verbose + self.prevalence_samples = None + + def aggregation_fit(self, classif_predictions, labels): + self.train_post = classif_predictions + if self.exact_train_prev: + self.train_prevalence = F.prevalence_from_labels(labels, classes=self.classes_) + else: + self.train_prevalence = F.prevalence_from_probabilities(classif_predictions) + self.ilr = _JaxILRTransformation() + return self + + def sample_from_posterior(self, classif_predictions): + n_test, n_classes = classif_predictions.shape + map_estimate, lam = mapls( + self.train_post, + test_probs=classif_predictions, + pz=self.train_prevalence, + return_lambda=True, + ) + + z0 = self.ilr(map_estimate) + if isinstance(self.prior, str) and self.prior in {'map', 'map2'}: + prior_context = { + "test_probs": classif_predictions, + "train_prev": self.train_prevalence, + "map_estimate": map_estimate, + } + alpha = _resolve_dirichlet_prior( + self.prior, + n_classes, + allow_mapls_priors=True, + n_test=n_test, + map_prev=prior_context, + map_lambda=lam, + ) + else: + alpha = _resolve_dirichlet_prior(self.prior, n_classes) + + mcmc = MCMC( + NUTS(self._numpyro_model), + num_warmup=self.num_warmup, + num_samples=self.num_samples, + num_chains=1, + progress_bar=self.verbose, + ) + mcmc.run( + jrandom.PRNGKey(self.mcmc_seed), + test_posteriors=classif_predictions, + alpha=alpha, + init_params={"z": z0} if self.mapls_chain_init else None, + ) + + samples = mcmc.get_samples()["z"] + self.prevalence_samples = np.asarray(self.ilr.inverse(samples)) + return self.prevalence_samples + + def aggregate(self, classif_predictions: np.ndarray): + return self.sample_from_posterior(classif_predictions).mean(axis=0) + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + confidence_level = self.confidence_level if confidence_level is None else confidence_level + classif_predictions = self.classify(instances) + point_estimate = self.aggregate(classif_predictions) + region = WithConfidenceABC.construct_region( + self.prevalence_samples, + confidence_level=confidence_level, + method=self.region, + ) + return point_estimate, region + + def _log_likelihood(self, test_classif, test_prev, train_prev): + log_w = jnp.log(test_prev) - jnp.log(train_prev) + return jnp.sum(jax_logsumexp(jnp.log(test_classif) + log_w, axis=-1)) + + def _numpyro_model(self, test_posteriors, alpha): + test_posteriors = jnp.asarray(test_posteriors) + n_classes = test_posteriors.shape[1] + + z = numpyro.sample("z", dist.Normal(jnp.zeros(n_classes - 1), 1.0)) + prev = self.ilr.inverse(z) + train_prev = jnp.asarray(self.train_prevalence) + alpha = jnp.asarray(alpha) + + numpyro.factor("dirichlet_prior", dist.Dirichlet(alpha).log_prob(prev)) + numpyro.factor( + "likelihood", + (1.0 / self.temperature) * self._log_likelihood(test_posteriors, test_prev=prev, train_prev=train_prev), + ) + + +def mapls( + train_probs: np.ndarray, + test_probs: np.ndarray, + pz: np.ndarray, + qy_mode: str = 'soft', + max_iter: int = 100, + init_mode: str = 'identical', + lam: float = None, + dvg_name='kl', + return_lambda=False, +): + cls_num = len(pz) + assert test_probs.shape[-1] == cls_num + if not isinstance(max_iter, int) or max_iter < 0: + raise ValueError(f'expected a non-negative integer for max_iter; found {max_iter!r}') + + if dvg_name == 'kl': + dvg = kl_div + elif dvg_name == 'js': + dvg = js_div + else: + raise ValueError(f'Unsupported distribution distance measure {dvg_name!r}; expected "kl" or "js"') + + q_prior = np.ones(cls_num) / cls_num + if lam is None: + lam = get_lambda(test_probs, pz, q_prior, dvg=dvg, max_iter=max_iter) + + qz = mapls_em( + test_probs, + pz, + lam, + q_prior, + cls_num, + init_mode=init_mode, + max_iter=max_iter, + qy_mode=qy_mode, + ) + return (qz, lam) if return_lambda else qz + + +def mapls_em(probs, pz, lam, q_prior, cls_num, init_mode='identical', max_iter=100, qy_mode='soft'): + pz = np.asarray(pz, dtype=float) + pz = pz / np.sum(pz) + if init_mode == 'uniform': + qz = np.ones(cls_num) / cls_num + elif init_mode == 'identical': + qz = pz.copy() + else: + raise ValueError('init_mode should be either "uniform" or "identical"') + + w = qz / pz + for _ in range(max_iter): + mapls_probs = normalized(probs * w, axis=-1, order=1) + if qy_mode == 'hard': + pred = np.argmax(mapls_probs, axis=-1) + qz_new = np.bincount(pred.reshape(-1), minlength=cls_num) + elif qy_mode == 'soft': + qz_new = np.mean(mapls_probs, axis=0) + else: + raise ValueError('qy_mode should be either "soft" or "hard"') + + qz = lam * qz_new + (1 - lam) * q_prior + qz /= qz.sum() + w = qz / pz + + return qz + + +def get_lambda(test_probs, pz, q_prior, dvg, max_iter=50): + n_classes = len(pz) + qz_pred = mapls_em(test_probs, pz, 1, 0, n_classes, max_iter=max_iter) + + tu_div = dvg(qz_pred, q_prior) + ts_div = dvg(qz_pred, pz) + su_div = dvg(pz, q_prior) + + su_conf = 1 - lambda_forward(su_div, lambda_inverse(dpq=0.5, lam=0.2)) + tu_conf = lambda_forward(tu_div, lambda_inverse(dpq=0.5, lam=su_conf)) + ts_conf = lambda_forward(ts_div, lambda_inverse(dpq=0.5, lam=su_conf)) + + confs = np.array([tu_conf, 1 - ts_conf]) + weights = np.array([0.9, 0.1]) + return np.sum(weights * confs) + + +def lambda_inverse(dpq, lam): + return (1 / (1 - lam) - 1) / dpq + + +def lambda_forward(dpq, gamma): + return gamma * dpq / (1 + gamma * dpq) + + +def get_lamda(test_probs, pz, q_prior, dvg, max_iter=50): + return get_lambda(test_probs, pz, q_prior, dvg, max_iter=max_iter) + + +def lam_inv(dpq, lam): + return lambda_inverse(dpq, lam) + + +def lam_forward(dpq, gamma): + return lambda_forward(dpq, gamma) + + +def kl_div(p, q, eps=1e-12): + p = np.asarray(p, dtype=float) + q = np.asarray(q, dtype=float) + + mask = p > 0 + return np.sum(p[mask] * np.log(p[mask] / (q[mask] + eps))) + + +def js_div(p, q): + assert (np.abs(np.sum(p) - 1) < 1e-6) and (np.abs(np.sum(q) - 1) < 1e-6) + m = (p + q) / 2 + return kl_div(p, m) / 2 + kl_div(q, m) / 2 + + +def normalized(a, axis=-1, order=2): + l2 = np.atleast_1d(np.linalg.norm(a, order, axis)) + l2[l2 == 0] = 1 + return a / np.expand_dims(l2, axis) + + +def alpha0_from_lambda(lam, n_test, n_classes): + return 1 + n_test * (1 - lam) / (lam * n_classes) + + +def alpha0_from_lamda(lam, n_test, n_classes): + return alpha0_from_lambda(lam, n_test, n_classes) + + +def calibrate_temperature( + method: WithConfidenceABC, + train: LabelledCollection, + val_prot: AbstractProtocol, + temp_grid=(0.5, 1.0, 1.5, 2.0, 5.0, 10.0, 100.0), + nominal_coverage: float = 0.95, + amplitude_threshold=1.0, + criterion: str = 'winkler', + n_jobs: int = 1, + verbose: bool = True, +): + """ + Calibrates the temperature parameter of a Bayesian quantifier with + confidence regions by selecting the value that yields the best validation + trade-off between nominal coverage and region sharpness. + + The method is first fitted on ``train``. For each candidate temperature, + the fitted quantifier is deep-copied, its ``temperature`` attribute is + replaced, and it is evaluated on the samples generated by ``val_prot``. + Candidate temperatures whose average region amplitude exceeds + ``amplitude_threshold`` are discarded. + + When ``criterion='winkler'``, the surviving candidate with minimum mean + Winkler score is selected. When ``criterion='auto'``, the selected + temperature is the one whose empirical coverage is closest to + ``nominal_coverage``. + + :param method: a quantifier implementing :class:`WithConfidenceABC` and + exposing a writable ``temperature`` attribute + :param train: training set used to fit the quantifier + :param val_prot: validation protocol yielding pairs ``(sample, true_prev)`` + :param temp_grid: candidate temperatures to evaluate + :param nominal_coverage: target confidence level used by the Winkler score + and coverage selection + :param amplitude_threshold: maximum allowed average simplex proportion of + the region. It can also be set to ``'auto'`` to use a heuristic based + on the number of classes + :param criterion: either ``'winkler'`` (default) or ``'auto'`` + :param n_jobs: number of parallel jobs across candidate temperatures + :param verbose: whether to display progress information + :return: the selected temperature value + """ + if not hasattr(method, 'temperature'): + raise ValueError(f'{method.__class__.__name__} does not expose a temperature attribute') + if not isinstance(method, WithConfidenceABC): + raise TypeError(f'{method.__class__.__name__} is not an instance of WithConfidenceABC') + if not 0 < nominal_coverage < 1: + raise ValueError(f'{nominal_coverage=} must be in the interval (0,1)') + if criterion not in {'auto', 'winkler'}: + raise ValueError(f'unknown {criterion=}; valid ones are "auto" or "winkler"') + if amplitude_threshold != 'auto': + if not isinstance(amplitude_threshold, Real) or amplitude_threshold > 1.0: + raise ValueError( + f'wrong value for {amplitude_threshold=}; it must either be "auto" or a real value <= 1.0' + ) + temperatures = sorted(_validate_temperature(temp) for temp in temp_grid) + + if amplitude_threshold == 'auto': + amplitude_threshold = 0.1 / np.log(train.n_classes + 1) + + if amplitude_threshold > 0.1: + print(f'warning: the {amplitude_threshold=} is too large; this may lead to uninformative regions') + + def _evaluate_temperature_job(job_id, temperature): + local_method = copy.deepcopy(method) + local_method.temperature = temperature + + coverage = 0 + amplitudes = [] + winklers = [] + errors = [] + + pbar = tqdm( + enumerate(val_prot()), + position=job_id, + total=val_prot.total(), + disable=not verbose, + ) + + for i, (sample, prev) in pbar: + point_estim, conf_region = local_method.predict_conf(sample) + + if prev in conf_region: + coverage += 1 + + amplitudes.append(conf_region.montecarlo_proportion(n_trials=50_000)) + if criterion == 'winkler': + winklers.append(conf_region.mean_winkler_score(true_prev=prev, alpha=1 - nominal_coverage)) + errors.append(qp.error.mae(prev, point_estim)) + + description = ( + f'job={job_id} T={temperature}: ' + f'MAE={np.mean(errors):.6f} ' + f'coverage={coverage / (i + 1) * 100:.2f}% ' + f'amplitude={np.mean(amplitudes) * 100:.4f}% ' + ) + if criterion == 'winkler': + description += f'winkler={np.mean(winklers):.4f}' + pbar.set_description(description) + + mean_coverage = coverage / val_prot.total() + mean_amplitude = float(np.mean(amplitudes)) + mean_winkler = float(np.mean(winklers)) if criterion == 'winkler' else None + mean_error = float(np.mean(errors)) + return temperature, mean_coverage, mean_amplitude, mean_winkler, mean_error + + method.fit(*train.Xy) + raw_results = Parallel(n_jobs=n_jobs, backend="loky")( + delayed(_evaluate_temperature_job)(job_id, temperature) + for job_id, temperature in tqdm(enumerate(temperatures), disable=not verbose) + ) + filtered_results = [ + (temperature, coverage, amplitude, winkler, error) + for temperature, coverage, amplitude, winkler, error in raw_results + if amplitude < amplitude_threshold + ] + + chosen_temperature = 1.0 + chosen_coverage = chosen_amplitude = chosen_winkler = chosen_error = None + + if filtered_results: + if criterion == 'winkler': + chosen_temperature, chosen_coverage, chosen_amplitude, chosen_winkler, chosen_error = min( + filtered_results, key=lambda item: item[3] + ) + else: + chosen_temperature, chosen_coverage, chosen_amplitude, chosen_winkler, chosen_error = min( + filtered_results, key=lambda item: abs(item[1] - nominal_coverage) + ) + + if verbose and chosen_coverage is not None: + message = ( + f'\nChosen_temperature={chosen_temperature:.2f} got ' + f'MAE={chosen_error:.6f} ' + f'coverage={chosen_coverage * 100:.2f}% ' + f'amplitude={chosen_amplitude * 100:.4f}% ' + ) + if criterion == 'winkler': + message += f'winkler={chosen_winkler:.4f}' + print(message) + + return chosen_temperature + + +def temp_calibration(*args, **kwargs): + """ + Backward-compatible alias for :func:`calibrate_temperature`. + """ + return calibrate_temperature(*args, **kwargs) diff --git a/quapy/method/_kdey.py b/quapy/method/_kdey.py index f004c1a..5445b34 100644 --- a/quapy/method/_kdey.py +++ b/quapy/method/_kdey.py @@ -1,11 +1,12 @@ import numpy as np +from numbers import Real from sklearn.base import BaseEstimator from sklearn.neighbors import KernelDensity import quapy as qp from quapy.method.aggregative import AggregativeSoftQuantifier import quapy.functional as F - +from scipy.special import logsumexp from sklearn.metrics.pairwise import rbf_kernel @@ -15,44 +16,57 @@ class KDEBase: """ BANDWIDTH_METHOD = ['scott', 'silverman'] + KERNELS = ['gaussian', 'aitchison', 'ilr'] @classmethod - def _check_bandwidth(cls, bandwidth): + def _check_bandwidth(cls, bandwidth, kernel): """ Checks that the bandwidth parameter is correct :param bandwidth: either a string (see BANDWIDTH_METHOD) or a float :return: the bandwidth if the check is passed, or raises an exception for invalid values """ - assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, float), \ + assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, Real), \ f'invalid bandwidth, valid ones are {KDEBase.BANDWIDTH_METHOD} or float values' - if isinstance(bandwidth, float): - assert 0 < bandwidth < 1, \ - "the bandwith for KDEy should be in (0,1), since this method models the unit simplex" + if isinstance(bandwidth, Real): + bandwidth = float(bandwidth) return bandwidth - def get_kde_function(self, X, bandwidth): + @classmethod + def _check_kernel(cls, kernel): + assert kernel in KDEBase.KERNELS, f'unknown {kernel=}' + return kernel + + def get_kde_function(self, X, bandwidth, kernel): """ Wraps the KDE function from scikit-learn. :param X: data for which the density function is to be estimated :param bandwidth: the bandwidth of the kernel + :param kernel: the kernel family :return: a scikit-learn's KernelDensity object """ + X = self.transform_posteriors(X, kernel) + bandwidth = self.effective_bandwidth(bandwidth, kernel) return KernelDensity(bandwidth=bandwidth).fit(X) - def pdf(self, kde, X): + def pdf(self, kde, X, kernel, log_densities=False): """ Wraps the density evalution of scikit-learn's KDE. Scikit-learn returns log-scores (s), so this function returns :math:`e^{s}` :param kde: a previously fit KDE function :param X: the data for which the density is to be estimated + :param kernel: the kernel family :return: np.ndarray with the densities """ - return np.exp(kde.score_samples(X)) + X = self.transform_posteriors(X, kernel) + log_density = kde.score_samples(X) + if log_densities: + return log_density + return np.exp(log_density) - def get_mixture_components(self, X, y, classes, bandwidth): + def get_mixture_components(self, X, y, classes, bandwidth, kernel): """ Returns an array containing the mixture components, i.e., the KDE functions for each class. @@ -60,15 +74,50 @@ class KDEBase: :param y: the class labels :param n_classes: integer, the number of classes :param bandwidth: float, the bandwidth of the kernel + :param kernel: the kernel family :return: a list of KernelDensity objects, each fitted with the corresponding class-specific covariates """ class_cond_X = [] for cat in classes: selX = X[y==cat] if selX.size==0: - selX = [F.uniform_prevalence(len(classes))] + raise ValueError(f'empty class {cat}') class_cond_X.append(selX) - return [self.get_kde_function(X_cond_yi, bandwidth) for X_cond_yi in class_cond_X] + return [self.get_kde_function(X_cond_yi, bandwidth, kernel) for X_cond_yi in class_cond_X] + + def transform_posteriors(self, X, kernel): + if kernel in {'aitchison', 'ilr'}: + X = self.shrink_posteriors(X) + if kernel == 'aitchison': + return self.clr_transform(X) + if kernel == 'ilr': + return self.ilr_transform(X) + return X + + def shrink_posteriors(self, X): + shrinkage = getattr(self, 'shrinkage', 0.0) + if shrinkage <= 0: + return X + X = np.asarray(X) + n_classes = X.shape[-1] + uniform = np.full(n_classes, 1.0 / n_classes, dtype=X.dtype) + return (1.0 - shrinkage) * X + shrinkage * uniform + + def effective_bandwidth(self, bandwidth, kernel): + shrinkage = getattr(self, 'shrinkage', 0.0) + if shrinkage > 0 and kernel in {'aitchison', 'ilr'} and isinstance(bandwidth, Real): + return (1.0 - shrinkage) * float(bandwidth) + return bandwidth + + def clr_transform(self, X): + if not hasattr(self, 'clr'): + self.clr = F.CLRtransformation() + return self.clr(X) + + def ilr_transform(self, X): + if not hasattr(self, 'ilr'): + self.ilr = F.ILRtransformation() + return self.ilr(X) class KDEyML(AggregativeSoftQuantifier, KDEBase): @@ -107,17 +156,31 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value for `k`); or as a tuple (X,y) defining the specific set of data to use for validation. :param bandwidth: float, the bandwidth of the Kernel + :param kernel: kernel of KDE, valid ones are in KDEBase.KERNELS + :param shrinkage: amount of shrinkage towards the uniform distribution to apply before + Aitchison/ILR transformations. Must be in ``[0,1)``. :param random_state: a seed to be set before fitting any base quantifier (default None) """ def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1, - random_state=None): + kernel='gaussian', shrinkage=0.0, random_state=None): super().__init__(classifier, fit_classifier, val_split) - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel) + self.kernel = self._check_kernel(kernel) + assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)' + assert self.kernel != 'gaussian' or shrinkage == 0, \ + 'shrinkage is only supported for Aitchison/ILR kernels' + self.shrinkage = float(shrinkage) self.random_state=random_state def aggregation_fit(self, classif_predictions, labels): - self.mix_densities = self.get_mixture_components(classif_predictions, labels, self.classes_, self.bandwidth) + self.mix_densities = self.get_mixture_components( + classif_predictions, + labels, + self.classes_, + self.bandwidth, + self.kernel, + ) return self def aggregate(self, posteriors: np.ndarray): @@ -129,15 +192,25 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): :return: a vector of class prevalence estimates """ with qp.util.temp_seed(self.random_state): - epsilon = 1e-10 + epsilon = 1e-12 n_classes = len(self.mix_densities) - test_densities = [self.pdf(kde_i, posteriors) for kde_i in self.mix_densities] + if (self.kernel != 'gaussian' and n_classes >= 20) or n_classes >= 30: + test_log_densities = [ + self.pdf(kde_i, posteriors, self.kernel, log_densities=True) + for kde_i in self.mix_densities + ] - def neg_loglikelihood(prev): - # test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) - test_mixture_likelihood = prev @ test_densities - test_loglikelihood = np.log(test_mixture_likelihood + epsilon) - return -np.sum(test_loglikelihood) + def neg_loglikelihood(prev): + prev = qp.error.smooth(prev, eps=epsilon) + test_loglikelihood = logsumexp(np.log(prev)[:, None] + test_log_densities, axis=0) + return -np.sum(test_loglikelihood) + else: + test_densities = [self.pdf(kde_i, posteriors, self.kernel) for kde_i in self.mix_densities] + + def neg_loglikelihood(prev): + test_mixture_likelihood = prev @ test_densities + test_loglikelihood = np.log(test_mixture_likelihood + epsilon) + return -np.sum(test_loglikelihood) return F.optim_minimize(neg_loglikelihood, n_classes) @@ -192,18 +265,22 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): super().__init__(classifier, fit_classifier, val_split) self.divergence = divergence - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian') self.random_state=random_state self.montecarlo_trials = montecarlo_trials def aggregation_fit(self, classif_predictions, labels): - self.mix_densities = self.get_mixture_components(classif_predictions, labels, self.classes_, self.bandwidth) + self.mix_densities = self.get_mixture_components( + classif_predictions, labels, self.classes_, self.bandwidth, 'gaussian' + ) N = self.montecarlo_trials rs = self.random_state n = len(self.classes_) self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.mix_densities]) - self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.mix_densities]) + self.reference_classwise_densities = np.asarray( + [self.pdf(kde_j, self.reference_samples, 'gaussian') for kde_j in self.mix_densities] + ) self.reference_density = np.mean(self.reference_classwise_densities, axis=0) # equiv. to (uniform @ self.reference_classwise_densities) return self @@ -213,8 +290,8 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): # apply importance sampling (IS). In this version we compute D(p_alpha||q) with IS n_classes = len(self.mix_densities) - test_kde = self.get_kde_function(posteriors, self.bandwidth) - test_densities = self.pdf(test_kde, self.reference_samples) + test_kde = self.get_kde_function(posteriors, self.bandwidth, 'gaussian') + test_densities = self.pdf(test_kde, self.reference_samples, 'gaussian') def f_squared_hellinger(u): return (np.sqrt(u)-1)**2 @@ -279,7 +356,7 @@ class KDEyCS(AggregativeSoftQuantifier): def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1): super().__init__(classifier, fit_classifier, val_split) - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian') def gram_matrix_mix_sum(self, X, Y=None): # this adapts the output of the rbf_kernel function (pairwise evaluations of Gaussian kernels k(x,y)) @@ -354,4 +431,3 @@ class KDEyCS(AggregativeSoftQuantifier): return partA + partB #+ partC return F.optim_minimize(divergence, n) - diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index 52c3720..5806b1d 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -3,7 +3,6 @@ from argparse import ArgumentError from copy import deepcopy from typing import Callable, Literal, Union import numpy as np -from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling from numpy.f2py.crackfortran import true_intent_list from sklearn.base import BaseEstimator from sklearn.calibration import CalibratedClassifierCV @@ -18,13 +17,27 @@ from quapy.functional import get_divergence from quapy.classification.svmperf import SVMperf from quapy.data import LabelledCollection from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric -from quapy.method import _bayesian # import warnings # from sklearn.exceptions import ConvergenceWarning # warnings.filterwarnings("ignore", category=ConvergenceWarning) +def _get_abstention_calibrators(): + try: + from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling + except ImportError as exc: + raise ImportError( + "Posterior calibration for EMQ requires the optional 'abstention' package." + ) from exc + return { + 'nbvs': NoBiasVectorScaling(), + 'bcts': TempScaling(bias_positions='all'), + 'ts': TempScaling(), + 'vs': VectorScaling(), + } + + # Abstract classes # ------------------------------------ @@ -849,12 +862,7 @@ class EMQ(AggregativeSoftQuantifier): "validation data") if self.calib is not None: - calibrator = { - 'nbvs': NoBiasVectorScaling(), - 'bcts': TempScaling(bias_positions='all'), - 'ts': TempScaling(), - 'vs': VectorScaling() - }.get(self.calib, None) + calibrator = _get_abstention_calibrators().get(self.calib, None) if calibrator is None: raise ValueError(f'invalid value for {self.calib=}; valid ones are {EMQ.CALIB_OPTIONS}') diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index c997fb3..c7ac218 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -1,24 +1,34 @@ +from numbers import Number +from typing import Iterable + import numpy as np +from joblib import Parallel, delayed from sklearn.base import BaseEstimator from sklearn.metrics import confusion_matrix import quapy as qp import quapy.functional as F -from quapy.method import _bayesian +from quapy.functional import CompositionalTransformation, CLRtransformation, ILRtransformation from quapy.data import LabelledCollection from quapy.method.aggregative import AggregativeQuantifier, AggregativeCrispQuantifier, AggregativeSoftQuantifier, BinaryAggregativeQuantifier from scipy.stats import chi2 from sklearn.utils import resample from abc import ABC, abstractmethod -from scipy.special import softmax, factorial +from scipy.special import factorial import copy from functools import lru_cache +from tqdm import tqdm """ This module provides implementation of different types of confidence regions, and the implementation of Bootstrap for AggregativeQuantifiers. """ + +def _get_bayesian_module(): + from quapy.method import _bayesian + return _bayesian + class ConfidenceRegionABC(ABC): """ Abstract class of confidence regions @@ -85,12 +95,58 @@ class ConfidenceRegionABC(ABC): """ Returns internal samples """ ... + def __contains__(self, p): + """ + Overloads in operator, checks if `p` is contained in the region + + :param p: array-like + :return: boolean + """ + p = np.asarray(p) + assert p.ndim==1, f'unexpected shape for point parameter' + return self.coverage(p)==1. + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + """ + Finds the closes point to p that belongs to the region. Assumes the region is convex. + + :param p: array-like, the point + :param tol: float, error tolerance + :param max_iter: int, max number of iterations + :returns: array-like, the closes point to p in the segment between p and the center of the region, that + belongs to the region + """ + p = np.asarray(p, dtype=float) + + # if p in region, returns p itself + if p in self: + return p.copy() + + # center of the region + c = self.point_estimate() + + # binary search in [0,1], interpolation parameter + # low=closest to p, high=closest to c + low, high = 0.0, 1.0 + for _ in range(max_iter): + mid = 0.5 * (low + high) + x = p*(1-mid) + c*mid + if x in self: + high = mid + else: + low = mid + if high - low < tol: + break + + in_boundary = p*(1-high) + c*high + return in_boundary + class WithConfidenceABC(ABC): """ Abstract class for confidence regions. """ - METHODS = ['intervals', 'ellipse', 'ellipse-clr'] + REGION_TYPE = ['intervals', 'ellipse', 'ellipse-clr', 'ellipse-ilr'] @abstractmethod def predict_conf(self, instances, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): @@ -118,7 +174,7 @@ class WithConfidenceABC(ABC): return self.predict_conf(instances=instances, confidence_level=confidence_level) @classmethod - def construct_region(cls, prev_estims, confidence_level=0.95, method='intervals'): + def construct_region(cls, prev_estims, confidence_level=0.95, method='intervals')->ConfidenceRegionABC: """ Construct a confidence region given many prevalence estimations. @@ -136,6 +192,8 @@ class WithConfidenceABC(ABC): region = ConfidenceEllipseSimplex(prev_estims, confidence_level=confidence_level) elif method == 'ellipse-clr': region = ConfidenceEllipseCLR(prev_estims, confidence_level=confidence_level) + elif method == 'ellipse-ilr': + region = ConfidenceEllipseILR(prev_estims, confidence_level=confidence_level) if region is None: raise NotImplementedError(f'unknown method {method}') @@ -153,7 +211,7 @@ def simplex_volume(n): return 1 / factorial(n) -def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): +def within_ellipse_prop__(values, mean, prec_matrix, chi2_critical): """ Checks the proportion of values that belong to the ellipse with center `mean` and precision matrix `prec_matrix` at a distance `chi2_critical`. @@ -186,102 +244,91 @@ def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): return within_elipse * 1.0 -class ConfidenceEllipseSimplex(ConfidenceRegionABC): +def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): """ - Instantiates a Confidence Ellipse in the probability simplex. + Checks the proportion of values that belong to the ellipse with center `mean` and precision matrix `prec_matrix` + at a distance `chi2_critical`. - :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) - :param confidence_level: float, the confidence level (default 0.95) + :param values: a np.ndarray of shape (n_dim,) or (n_values, n_dim,) + :param mean: a np.ndarray of shape (n_dim,) with the center of the ellipse + :param prec_matrix: a np.ndarray with the precision matrix (inverse of the + covariance matrix) of the ellipse. If this inverse cannot be computed + then None must be passed + :param chi2_critical: float, the chi2 critical value + + :return: float in [0,1], the fraction of values that are contained in the ellipse + defined by the mean (center), the precision matrix (shape), and the chi2_critical value (distance). + If `values` is only one value, then either 0. (not contained) or 1. (contained) is returned. + """ + if prec_matrix is None: + return 0. + + values = np.atleast_2d(values) + diff = values - mean + d_M_squared = np.sum(diff @ prec_matrix * diff, axis=-1) + within_ellipse = d_M_squared <= chi2_critical + + if len(within_ellipse) == 1: + return float(within_ellipse[0]) + else: + return float(np.mean(within_ellipse)) + + +def closest_point_on_ellipsoid(p, mean, cov, chi2_critical, tol=1e-9, max_iter=100): + """ + Computes the closest point on the ellipsoid defined by: + (x - mean)^T cov^{-1} (x - mean) = chi2_critical """ - def __init__(self, samples, confidence_level=0.95): + p = np.asarray(p) + mean = np.asarray(mean) + Sigma = np.asarray(cov) - assert 0. < confidence_level < 1., f'{confidence_level=} must be in range(0,1)' + # Precompute precision matrix + P = np.linalg.pinv(Sigma) + d = P.shape[0] - samples = np.asarray(samples) + # Define v = p - mean + v = p - mean - self.mean_ = samples.mean(axis=0) - self.cov_ = np.cov(samples, rowvar=False, ddof=1) + # If p is inside the ellipsoid, return p itself + M_dist = v @ P @ v + if M_dist <= chi2_critical: + return p.copy() - try: - self.precision_matrix_ = np.linalg.inv(self.cov_) - except: - self.precision_matrix_ = None + # Function to compute x(lambda) + def x_lambda(lmbda): + A = np.eye(d) + lmbda * P + return mean + np.linalg.solve(A, v) - self.dim = samples.shape[-1] - self.ddof = self.dim - 1 + # Function whose root we want: f(lambda) = Mahalanobis distance - chi2 + def f(lmbda): + x = x_lambda(lmbda) + diff = x - mean + return diff @ P @ diff - chi2_critical - # critical chi-square value - self.confidence_level = confidence_level - self.chi2_critical_ = chi2.ppf(confidence_level, df=self.ddof) - self._samples = samples + # Bisection search over lambda >= 0 + l_low, l_high = 0.0, 1.0 - @property - def samples(self): - return self._samples + # Increase high until f(high) < 0 + while f(l_high) > 0: + l_high *= 2 + if l_high > 1e12: + raise RuntimeError("Failed to bracket the root.") - def point_estimate(self): - """ - Returns the point estimate, the center of the ellipse. + # Bisection + for _ in range(max_iter): + l_mid = 0.5 * (l_low + l_high) + fm = f(l_mid) + if abs(fm) < tol: + break + if fm > 0: + l_low = l_mid + else: + l_high = l_mid - :return: np.ndarray of shape (n_classes,) - """ - return self.mean_ - - def coverage(self, true_value): - """ - Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the - fraction of these that are contained in the region, if more than one value is passed. If only one value is - passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. - - :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) - :return: float in [0,1] - """ - return within_ellipse_prop(true_value, self.mean_, self.precision_matrix_, self.chi2_critical_) - - -class ConfidenceEllipseCLR(ConfidenceRegionABC): - """ - Instantiates a Confidence Ellipse in the Centered-Log Ratio (CLR) space. - - :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) - :param confidence_level: float, the confidence level (default 0.95) - """ - - def __init__(self, samples, confidence_level=0.95): - samples = np.asarray(samples) - self.clr = CLRtransformation() - Z = self.clr(samples) - self.mean_ = np.mean(samples, axis=0) - self.conf_region_clr = ConfidenceEllipseSimplex(Z, confidence_level=confidence_level) - self._samples = samples - - @property - def samples(self): - return self._samples - - def point_estimate(self): - """ - Returns the point estimate, the center of the ellipse. - - :return: np.ndarray of shape (n_classes,) - """ - # The inverse of the CLR does not coincide with the true mean, because the geometric mean - # requires smoothing the prevalence vectors and this affects the softmax (inverse); - # return self.clr.inverse(self.mean_) # <- does not coincide - return self.mean_ - - def coverage(self, true_value): - """ - Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the - fraction of these that are contained in the region, if more than one value is passed. If only one value is - passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. - - :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) - :return: float in [0,1] - """ - transformed_values = self.clr(true_value) - return self.conf_region_clr.coverage(transformed_values) + l_opt = l_mid + return x_lambda(l_opt) class ConfidenceIntervals(ConfidenceRegionABC): @@ -290,18 +337,30 @@ class ConfidenceIntervals(ConfidenceRegionABC): :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. """ - def __init__(self, samples, confidence_level=0.95): + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): assert 0 < confidence_level < 1, f'{confidence_level=} must be in range(0,1)' + assert samples.ndim == 2, 'unexpected shape; must be (n_bootstrap_samples, n_classes)' samples = np.asarray(samples) self.means_ = samples.mean(axis=0) + self.confidence_level = confidence_level alpha = 1-confidence_level + if bonferroni_correction: + n_classes = samples.shape[-1] + if n_classes>2: + alpha = alpha/n_classes low_perc = (alpha/2.)*100 high_perc = (1-alpha/2.)*100 self.I_low, self.I_high = np.percentile(samples, q=[low_perc, high_perc], axis=0) self._samples = samples + self.alpha = alpha @property def samples(self): @@ -330,36 +389,278 @@ class ConfidenceIntervals(ConfidenceRegionABC): return proportion + def coverage_soft(self, true_value): + within_intervals = np.logical_and(self.I_low <= true_value, true_value <= self.I_high) + return np.mean(within_intervals.astype(float)) + def __repr__(self): return '['+', '.join(f'({low:.4f}, {high:.4f})' for (low,high) in zip(self.I_low, self.I_high))+']' + @property + def n_dim(self): + return len(self.I_low) -class CLRtransformation: + def winkler_scores(self, true_prev, alpha=None, add_ae=False): + true_prev = np.asarray(true_prev) + assert true_prev.ndim == 1, 'unexpected dimensionality for true_prev' + assert len(true_prev)==self.n_dim, \ + f'unexpected number of dimensions; found {true_prev.ndim}, expected {self.n_dim}' + + def winkler_score(low, high, true_val, alpha, center): + amp = high-low + scale_cost = 2./alpha + cost = np.max([0, low-true_val], axis=0) + np.max([0, true_val-high], axis=0) + err = 0 + if add_ae: + err = abs(true_val - center) + return amp + scale_cost*cost + err + + alpha = alpha or self.alpha + return np.asarray( + [winkler_score(low_i, high_i, true_v, alpha, center) + for (low_i, high_i, true_v, center) in zip(self.I_low, self.I_high, true_prev, self.point_estimate())] + ) + + def mean_winkler_score(self, true_prev, alpha=None, add_ae=False): + return np.mean(self.winkler_scores(true_prev, alpha=alpha, add_ae=add_ae)) + + + +class ConfidenceEllipseSimplex(ConfidenceRegionABC): """ - Centered log-ratio, from component analysis + Instantiates a Confidence Ellipse in the probability simplex. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) """ - def __call__(self, X, epsilon=1e-6): - """ - Applies the CLR function to X thus mapping the instances, which are contained in `\\mathcal{R}^{n}` but - actually lie on a `\\mathcal{R}^{n-1}` simplex, onto an unrestricted space in :math:`\\mathcal{R}^{n}` - :param X: np.ndarray of (n_instances, n_dimensions) to be transformed - :param epsilon: small float for prevalence smoothing - :return: np.ndarray of (n_instances, n_dimensions), the CLR-transformed points - """ - X = np.asarray(X) - X = qp.error.smooth(X, epsilon) - G = np.exp(np.mean(np.log(X), axis=-1, keepdims=True)) # geometric mean - return np.log(X / G) + def __init__(self, samples, confidence_level=0.95): - def inverse(self, X): - """ - Inverse function. However, clr.inverse(clr(X)) does not exactly coincide with X due to smoothing. + assert 0. < confidence_level < 1., f'{confidence_level=} must be in range(0,1)' - :param X: np.ndarray of (n_instances, n_dimensions) to be transformed - :return: np.ndarray of (n_instances, n_dimensions), the CLR-transformed points + samples = np.asarray(samples) + + self.confidence_level = confidence_level + self.mean_ = samples.mean(axis=0) + self.cov_ = np.cov(samples, rowvar=False, ddof=1) + + try: + self.precision_matrix_ = np.linalg.pinv(self.cov_) + except: + self.precision_matrix_ = None + + self.dim = samples.shape[-1] + self.ddof = self.dim - 1 + + # critical chi-square value + self.confidence_level = confidence_level + self.chi2_critical_ = chi2.ppf(confidence_level, df=self.ddof) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): """ - return softmax(X, axis=-1) + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + return within_ellipse_prop(true_value, self.mean_, self.precision_matrix_, self.chi2_critical_) + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + return closest_point_on_ellipsoid( + p, + mean=self.mean_, + cov=self.cov_, + chi2_critical=self.chi2_critical_ + ) + + +class ConfidenceEllipseTransformed(ConfidenceRegionABC): + """ + Instantiates a Confidence Ellipse in a transformed space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + + def __init__(self, samples, transformation: CompositionalTransformation, confidence_level=0.95): + samples = np.asarray(samples) + self.transformation = transformation + self.confidence_level = confidence_level + Z = self.transformation(samples) + self.mean_ = np.mean(samples, axis=0) + # self.mean_ = self.transformation.inverse(np.mean(Z, axis=0)) + self.conf_region_z = ConfidenceEllipseSimplex(Z, confidence_level=confidence_level) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): + """ + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + # The inverse of the CLR does not coincide with the true mean, because the geometric mean + # requires smoothing the prevalence vectors and this affects the softmax (inverse); + # return self.clr.inverse(self.mean_) # <- does not coincide + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage(transformed_values) + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + p_prime = self.transformation(p) + b_prime = self.conf_region_z.closest_point_in_region(p_prime, tol=tol, max_iter=max_iter) + b = self.transformation.inverse(b_prime) + return b + + +class ConfidenceEllipseCLR(ConfidenceEllipseTransformed): + """ + Instantiates a Confidence Ellipse in the Centered-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + def __init__(self, samples, confidence_level=0.95): + super().__init__(samples, CLRtransformation(), confidence_level=confidence_level) + + +class ConfidenceEllipseILR(ConfidenceEllipseTransformed): + """ + Instantiates a Confidence Ellipse in the Isometric-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + def __init__(self, samples, confidence_level=0.95): + super().__init__(samples, ILRtransformation(), confidence_level=confidence_level) + + + +class ConfidenceIntervalsTransformed(ConfidenceRegionABC): + """ + Instantiates a Confidence Interval region in a transformed space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + + def __init__(self, samples, transformation: CompositionalTransformation, confidence_level=0.95, bonferroni_correction=False): + samples = np.asarray(samples) + self.transformation = transformation + self.confidence_level = confidence_level + Z = self.transformation(samples) + self.mean_ = np.mean(samples, axis=0) + # self.mean_ = self.transformation.inverse(np.mean(Z, axis=0)) + self.conf_region_z = ConfidenceIntervals(Z, confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): + """ + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + # The inverse of the CLR does not coincide with the true mean, because the geometric mean + # requires smoothing the prevalence vectors and this affects the softmax (inverse); + # return self.clr.inverse(self.mean_) # <- does not coincide + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage(transformed_values) + + def coverage_soft(self, true_value): + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage_soft(transformed_values) + + def winkler_scores(self, true_prev, alpha=None, add_ae=False): + transformed_values = self.transformation(true_prev) + return self.conf_region_z.winkler_scores(transformed_values, alpha=alpha, add_ae=add_ae) + + def mean_winkler_score(self, true_prev, alpha=None, add_ae=False): + transformed_values = self.transformation(true_prev) + return self.conf_region_z.mean_winkler_score(transformed_values, alpha=alpha, add_ae=add_ae) + + +class ConfidenceIntervalsCLR(ConfidenceIntervalsTransformed): + """ + Instantiates a Confidence Intervals in the Centered-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): + super().__init__(samples, CLRtransformation(), confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + + +class ConfidenceIntervalsILR(ConfidenceIntervalsTransformed): + """ + Instantiates a Confidence Intervals in the Isometric-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): + super().__init__(samples, ILRtransformation(), confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): @@ -399,7 +700,8 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): n_test_samples=500, confidence_level=0.95, region='intervals', - random_state=None): + random_state=None, + verbose=False): assert isinstance(quantifier, AggregativeQuantifier), \ f'base quantifier does not seem to be an instance of {AggregativeQuantifier.__name__}' @@ -416,32 +718,45 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): self.confidence_level = confidence_level self.region = region self.random_state = random_state + self.verbose = verbose def aggregation_fit(self, classif_predictions, labels): - data = LabelledCollection(classif_predictions, labels, classes=self.classes_) + self.quantifiers = [] if self.n_train_samples==1: self.quantifier.aggregation_fit(classif_predictions, labels) self.quantifiers.append(self.quantifier) else: - # model-based bootstrap (only on the aggregative part) - n_examples = len(data) - full_index = np.arange(n_examples) - with qp.util.temp_seed(self.random_state): - for i in range(self.n_train_samples): - quantifier = copy.deepcopy(self.quantifier) - index = resample(full_index, n_samples=n_examples) - classif_predictions_i = classif_predictions.sampling_from_index(index) - data_i = data.sampling_from_index(index) - quantifier.aggregation_fit(classif_predictions_i, data_i) - self.quantifiers.append(quantifier) + if classif_predictions is None or labels is None: + # The entire dataset was consumed for classifier training, implying there is no need for training + # an aggregation function. If the bootstrap method was configured to train different aggregators + # (i.e., self.n_train_samples>1), then an error is raise. Otherwise, the method ends. + if self.n_train_samples > 1: + raise ValueError( + f'The underlying quantifier ({self.quantifier.__class__.__name__}) has consumed, all training ' + f'data, meaning the aggregation function needs none, but {self.n_train_samples=} is > 1, which ' + f'is inconsistent.' + ) + else: + # model-based bootstrap (only on the aggregative part) + data = LabelledCollection(classif_predictions, labels, classes=self.classes_) + n_examples = len(data) + full_index = np.arange(n_examples) + with qp.util.temp_seed(self.random_state): + for i in range(self.n_train_samples): + quantifier = copy.deepcopy(self.quantifier) + index = resample(full_index, n_samples=n_examples) + classif_predictions_i = classif_predictions.sampling_from_index(index) + data_i = data.sampling_from_index(index) + quantifier.aggregation_fit(classif_predictions_i, data_i) + self.quantifiers.append(quantifier) return self def aggregate(self, classif_predictions: np.ndarray): prev_mean, self.confidence = self.aggregate_conf(classif_predictions) return prev_mean - def aggregate_conf(self, classif_predictions: np.ndarray, confidence_level=None): + def aggregate_conf_sequential__(self, classif_predictions: np.ndarray, confidence_level=None): if confidence_level is None: confidence_level = self.confidence_level @@ -449,7 +764,7 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): prevs = [] with qp.util.temp_seed(self.random_state): for quantifier in self.quantifiers: - for i in range(self.n_test_samples): + for i in tqdm(range(self.n_test_samples), desc='resampling', total=self.n_test_samples, disable=not self.verbose): sample_i = resample(classif_predictions, n_samples=n_samples) prev_i = quantifier.aggregate(sample_i) prevs.append(prev_i) @@ -459,6 +774,26 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): return prev_estim, conf + def aggregate_conf(self, classif_predictions: np.ndarray, confidence_level=None): + confidence_level = confidence_level or self.confidence_level + + + n_samples = classif_predictions.shape[0] + prevs = [] + with qp.util.temp_seed(self.random_state): + for quantifier in self.quantifiers: + results = Parallel(n_jobs=-1)( + delayed(bootstrap_once)(i, classif_predictions, quantifier, n_samples) + for i in range(self.n_test_samples) + ) + prevs.extend(results) + + prevs = np.array(prevs) + conf = WithConfidenceABC.construct_region(prevs, confidence_level, method=self.region) + prev_estim = conf.point_estimate() + + return prev_estim, conf + def fit(self, X, y): self.quantifier._check_init_parameters() classif_predictions, labels = self.quantifier.classifier_fit_predict(X, y) @@ -477,6 +812,13 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): return self.quantifier._classifier_method() +def bootstrap_once(i, classif_predictions, quantifier, n_samples): + idx = np.random.randint(0, len(classif_predictions), n_samples) + sample = classif_predictions[idx] + prev = quantifier.aggregate(sample) + return prev + + class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): """ `Bayesian quantification `_ method (by Albert Ziegler and Paweł Czyż), @@ -506,6 +848,8 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): :param region: string, set to `intervals` for constructing confidence intervals (default), or to `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param prior: an array-like with the alpha parameters of a Dirichlet prior, a scalar real value + to be broadcast to all classes, or the string 'uniform' for a uniform, uninformative prior (default) """ def __init__(self, classifier: BaseEstimator=None, @@ -515,14 +859,21 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): num_samples: int = 1_000, mcmc_seed: int = 0, confidence_level: float = 0.95, - region: str = 'intervals'): + region: str = 'intervals', + temperature = 1., + prior = 'uniform'): if num_warmup <= 0: raise ValueError(f'parameter {num_warmup=} must be a positive integer') if num_samples <= 0: raise ValueError(f'parameter {num_samples=} must be a positive integer') + assert ((isinstance(prior, str) and prior == 'uniform') or + isinstance(prior, Number) or + (isinstance(prior, Iterable) and all(isinstance(v, Number) for v in prior))), \ + f'wrong type for {prior=}; expected "uniform", a real scalar, or an array-like of real values' - if _bayesian.DEPENDENCIES_INSTALLED is False: + bayesian = _get_bayesian_module() + if bayesian.DEPENDENCIES_INSTALLED is False: raise ImportError("Auxiliary dependencies are required. " "Run `$ pip install quapy[bayes]` to install them.") @@ -532,6 +883,8 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): self.mcmc_seed = mcmc_seed self.confidence_level = confidence_level self.region = region + self.temperature = temperature + self.prior = prior # Array of shape (n_classes, n_predicted_classes,) where entry (y, c) is the number of instances # labeled as class y and predicted as class c. @@ -562,11 +915,26 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): n_c_unlabeled = F.counts_from_labels(classif_predictions, self.classifier.classes_).astype(float) - self._samples = _bayesian.sample_posterior( + n_classes = len(self.classifier.classes_) + if isinstance(self.prior, str) and self.prior == 'uniform': + alpha = np.ones(n_classes, dtype=float) + elif isinstance(self.prior, Number): + alpha = np.full(n_classes, float(self.prior), dtype=float) + else: + alpha = np.asarray(self.prior, dtype=float) + if alpha.ndim != 1 or len(alpha) != n_classes: + raise ValueError( + f'wrong shape for prior; expected {n_classes} values, found shape {alpha.shape}' + ) + + bayesian = _get_bayesian_module() + self._samples = bayesian.sample_posterior_bayesianCC( n_c_unlabeled=n_c_unlabeled, n_y_and_c_labeled=self._n_and_c_labeled, num_warmup=self.num_warmup, num_samples=self.num_samples, + alpha=alpha, + temperature=self.temperature, seed=self.mcmc_seed, ) return self._samples @@ -574,15 +942,18 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): def get_prevalence_samples(self): if self._samples is None: raise ValueError("sample_from_posterior must be called before get_prevalence_samples") - return self._samples[_bayesian.P_TEST_Y] + bayesian = _get_bayesian_module() + return self._samples[bayesian.P_TEST_Y] def get_conditional_probability_samples(self): if self._samples is None: raise ValueError("sample_from_posterior must be called before get_conditional_probability_samples") - return self._samples[_bayesian.P_C_COND_Y] + bayesian = _get_bayesian_module() + return self._samples[bayesian.P_C_COND_Y] def aggregate(self, classif_predictions): - samples = self.sample_from_posterior(classif_predictions)[_bayesian.P_TEST_Y] + bayesian = _get_bayesian_module() + samples = self.sample_from_posterior(classif_predictions)[bayesian.P_TEST_Y] return np.asarray(samples.mean(axis=0), dtype=float) def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): @@ -637,17 +1008,19 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): if num_samples <= 0: raise ValueError(f'parameter {num_samples=} must be a positive integer') - if not _bayesian.DEPENDENCIES_INSTALLED: + bayesian = _get_bayesian_module() + if not bayesian.DEPENDENCIES_INSTALLED: raise ImportError("Auxiliary dependencies are required. " "Run `$ pip install quapy[bayes]` to install them.") super().__init__(classifier, fit_classifier, val_split) + self.nbins = nbins self.fixed_bins = fixed_bins self.num_warmup = num_warmup self.num_samples = num_samples self.stan_seed = stan_seed - self.stan_code = _bayesian.load_stan_file() + self.stan_code = bayesian.load_stan_file() self.confidence_level = confidence_level self.region = region @@ -676,7 +1049,8 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): def aggregate(self, classif_predictions): Px_test = classif_predictions[:, self.pos_label] test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) - prevs = _bayesian.pq_stan( + bayesian = _get_bayesian_module() + prevs = bayesian.pq_stan( self.stan_code, self.nbins, self.pos_hist, self.neg_hist, test_hist, self.num_samples, self.num_warmup, self.stan_seed ).flatten() @@ -694,5 +1068,3 @@ class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): predictions = self.classify(instances) return self.aggregate_conf(predictions, confidence_level=confidence_level) - - diff --git a/quapy/protocol.py b/quapy/protocol.py index 9a7e5c4..45efa6f 100644 --- a/quapy/protocol.py +++ b/quapy/protocol.py @@ -10,6 +10,8 @@ from quapy.data import LabelledCollection import quapy.functional as F from os.path import exists from glob import glob +from collections.abc import Iterable +from numbers import Number class AbstractProtocol(metaclass=ABCMeta): @@ -171,7 +173,7 @@ class AbstractStochasticSeededProtocol(AbstractProtocol): return sample -class OnLabelledCollectionProtocol: +class OnLabelledCollectionProtocol(AbstractStochasticSeededProtocol): """ Protocols that generate samples from a :class:`qp.data.LabelledCollection` object. """ @@ -229,8 +231,17 @@ class OnLabelledCollectionProtocol: elif return_type=='index': return lambda lc,params:params + def sample(self, index): + """ + Realizes the sample given the index of the instances. -class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): + :param index: indexes of the instances to select + :return: an instance of :class:`qp.data.LabelledCollection` + """ + return self.data.sampling_from_index(index) + + +class APP(OnLabelledCollectionProtocol): """ Implementation of the artificial prevalence protocol (APP). The APP consists of exploring a grid of prevalence values containing `n_prevalences` points (e.g., @@ -311,15 +322,6 @@ class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): - """ - Realizes the sample given the index of the instances. - - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` - """ - return self.data.sampling_from_index(index) - def total(self): """ Returns the number of samples that will be generated @@ -329,7 +331,7 @@ class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats) -class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): +class NPP(OnLabelledCollectionProtocol): """ A generator of samples that implements the natural prevalence protocol (NPP). The NPP consists of drawing samples uniformly at random, therefore approximately preserving the natural prevalence of the collection. @@ -365,15 +367,6 @@ class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): - """ - Realizes the sample given the index of the instances. - - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` - """ - return self.data.sampling_from_index(index) - def total(self): """ Returns the number of samples that will be generated (equals to "repeats") @@ -383,7 +376,7 @@ class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): return self.repeats -class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): +class UPP(OnLabelledCollectionProtocol): """ A variant of :class:`APP` that, instead of using a grid of equidistant prevalence values, relies on the Kraemer algorithm for sampling unit (k-1)-simplex uniformly at random, with @@ -423,14 +416,69 @@ class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): + def total(self): """ - Realizes the sample given the index of the instances. + Returns the number of samples that will be generated (equals to "repeats") - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` + :return: int """ - return self.data.sampling_from_index(index) + return self.repeats + + +class DirichletProtocol(OnLabelledCollectionProtocol): + """ + A protocol that establishes a prior Dirichlet distribution for the prevalence of the samples. + Note that providing an all-ones vector of Dirichlet parameters is equivalent to invoking the + APP protocol (although each protocol will generate a different series of samples given a + fixed seed, since the implementation is different). + + :param data: a `LabelledCollection` from which the samples will be drawn + :param alpha: an array-like of shape (n_classes,) with the parameters of the Dirichlet distribution + :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from + qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised. + :param repeats: the number of samples to generate. Default is 100. + :param random_state: allows replicating samples across runs (default 0, meaning that the sequence of samples + will be the same every time the protocol is called) + :param return_type: set to "sample_prev" (default) to get the pairs of (sample, prevalence) at each iteration, or + to "labelled_collection" to get instead instances of LabelledCollection + """ + + def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0, + return_type='sample_prev'): + #assert ((isinstance(alpha, str) and alpha == 'uniform') or + # isinstance(alpha, Number) or + # (isinstance(alpha, Iterable) and all(isinstance(v, Number) for v in alpha))), \ + # f'wrong type for {alpha=}; expected "uniform", a real scalar, or an array-like of real values' + + n_classes = data.n_classes + if isinstance(alpha, str) and alpha == 'uniform': + self.alpha = np.ones(n_classes, dtype=float) + elif isinstance(alpha, Number): + self.alpha = np.full(n_classes, float(alpha), dtype=float) + else: + self.alpha = np.asarray(alpha, dtype=float) + if self.alpha.ndim != 1 or len(self.alpha) != n_classes: + raise ValueError( + f'wrong shape for alpha; expected {n_classes} values, found shape {self.alpha.shape}' + ) + + super(DirichletProtocol, self).__init__(random_state) + self.data = data + #self.alpha = alpha + self.sample_size = qp._get_sample_size(sample_size) + self.repeats = repeats + self.random_state = random_state + self.collator = OnLabelledCollectionProtocol.get_collator(return_type) + + def samples_parameters(self): + """ + Return all the necessary parameters to replicate the samples. + + :return: a list of indexes that realize the sampling + """ + prevs = np.random.dirichlet(self.alpha, size=self.repeats) + indexes = [self.data.sampling_index(self.sample_size, *prevs_i) for prevs_i in prevs] + return indexes def total(self): """ @@ -450,7 +498,7 @@ class DomainMixer(AbstractStochasticSeededProtocol): :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised. :param repeats: int, number of samples to draw for every mixture rate - :param prevalence: the prevalence to preserv along the mixtures. If specified, should be an array containing + :param prevalence: the prevalence to preserve along the mixtures. If specified, should be an array containing one prevalence value (positive float) for each class and summing up to one. If not specified, the prevalence will be taken from the domain A (default). :param mixture_points: an integer indicating the number of points to take from a linear scale (e.g., 21 will diff --git a/quapy/tests/__init__.py b/quapy/tests/__init__.py index e69de29..374e4b0 100644 --- a/quapy/tests/__init__.py +++ b/quapy/tests/__init__.py @@ -0,0 +1,10 @@ +""" +Fast unit tests live in ``test_*.py`` and are intended to run without network +access or large external resources. + +Slow integration tests that download datasets or depend on optional stacks live +in ``integration_*.py`` and are meant to be run explicitly, e.g.: + + python -m unittest quapy.tests.integration_datasets + python -m unittest quapy.tests.integration_methods +""" diff --git a/quapy/tests/_synthetic.py b/quapy/tests/_synthetic.py new file mode 100644 index 0000000..84be4f2 --- /dev/null +++ b/quapy/tests/_synthetic.py @@ -0,0 +1,48 @@ +import numpy as np +from sklearn.datasets import make_classification + +from quapy.data import LabelledCollection +from quapy.data.base import Dataset + + +def make_labelled_collection( + n_samples=200, + n_features=12, + n_classes=2, + class_sep=1.5, + random_state=0, +): + n_informative = min(n_features, max(4, n_classes * 2)) + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_informative, + n_redundant=0, + n_repeated=0, + n_classes=n_classes, + n_clusters_per_class=1, + class_sep=class_sep, + random_state=random_state, + ) + classes = np.arange(n_classes) + return LabelledCollection(X, y, classes=classes) + + +def make_dataset( + n_train=150, + n_test=80, + n_features=12, + n_classes=2, + class_sep=1.5, + random_state=0, + name='synthetic', +): + data = make_labelled_collection( + n_samples=n_train + n_test, + n_features=n_features, + n_classes=n_classes, + class_sep=class_sep, + random_state=random_state, + ) + training, test = data.split_stratified(train_prop=n_train / (n_train + n_test), random_state=random_state) + return Dataset(training, test, name=name) diff --git a/quapy/tests/test_datasets.py b/quapy/tests/integration_datasets.py similarity index 75% rename from quapy/tests/test_datasets.py rename to quapy/tests/integration_datasets.py index de5f61a..1e8445b 100644 --- a/quapy/tests/test_datasets.py +++ b/quapy/tests/integration_datasets.py @@ -1,3 +1,10 @@ +""" +Integration tests for dataset fetchers and large external resources. + +This module is intentionally excluded from default ``unittest`` discovery by +using an ``integration_*.py`` filename. +""" + import os import unittest @@ -5,11 +12,11 @@ from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression import quapy.functional as F -from quapy.method.aggregative import PCC from quapy.data.datasets import * +from quapy.method.aggregative import PCC -class TestDatasets(unittest.TestCase): +class IntegrationDatasetsTest(unittest.TestCase): def new_quantifier(self): return PCC(LogisticRegression(C=0.001, max_iter=100)) @@ -17,13 +24,11 @@ class TestDatasets(unittest.TestCase): def _check_dataset(self, dataset): train, test = dataset.reduce().train_test q = self.new_quantifier() - print(f'testing method {q} in {dataset.name}...', end='') - if len(train)>500: + if len(train) > 500: train = train.sampling(500) q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.instances) self.assertTrue(F.check_prevalence_vector(estim_prevalences)) - print(f'[done]') def _check_samples(self, gen, q, max_samples_test=5, vectorizer=None): for X, p in gen(): @@ -37,54 +42,37 @@ class TestDatasets(unittest.TestCase): def test_reviews(self): for dataset_name in REVIEWS_SENTIMENT_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_reviews(dataset_name, tfidf=True, min_df=10) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_twitter(self): - # all the datasets are contained in the same resource; if the first one - # works, there is no need to test for the rest for dataset_name in TWITTER_SENTIMENT_DATASETS_TEST[:1]: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_twitter(dataset_name, min_df=10) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_UCIBinaryDataset(self): for dataset_name in UCI_BINARY_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_UCIBinaryDataset(dataset_name) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_UCIMultiDataset(self): for dataset_name in UCI_MULTICLASS_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_UCIMulticlassDataset(dataset_name) - dataset.stats() n_classes = dataset.n_classes uniform_prev = F.uniform_prevalence(n_classes) dataset.training = dataset.training.sampling(100, *uniform_prev) dataset.test = dataset.test.sampling(100, *uniform_prev) - print(f'[done]') self._check_dataset(dataset) def test_lequa2022(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_lequa2022 because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return for dataset_name in LEQUA2022_VECTOR_TASKS: - print(f'LeQu2022: loading dataset {dataset_name}...', end='') train, gen_val, gen_test = fetch_lequa2022(dataset_name) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() @@ -93,9 +81,7 @@ class TestDatasets(unittest.TestCase): self._check_samples(gen_test, q, max_samples_test=5) for dataset_name in LEQUA2022_TEXT_TASKS: - print(f'LeQu2022: loading dataset {dataset_name}...', end='') train, gen_val, gen_test = fetch_lequa2022(dataset_name) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) tfidf = TfidfVectorizer() @@ -107,13 +93,10 @@ class TestDatasets(unittest.TestCase): def test_lequa2024(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_lequa2024 because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return for task in LEQUA2024_TASKS: - print(f'LeQu2024: loading task {task}...', end='') train, gen_val, gen_test = fetch_lequa2024(task, merge_T3=True) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() @@ -121,16 +104,12 @@ class TestDatasets(unittest.TestCase): self._check_samples(gen_val, q, max_samples_test=5) self._check_samples(gen_test, q, max_samples_test=5) - def test_IFCB(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_IFCB because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return - print(f'loading dataset IFCB.') for mod_sel in [False, True]: train, gen = fetch_IFCB(single_sample_train=True, for_model_selection=mod_sel) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() diff --git a/quapy/tests/integration_methods.py b/quapy/tests/integration_methods.py new file mode 100644 index 0000000..180d80e --- /dev/null +++ b/quapy/tests/integration_methods.py @@ -0,0 +1,42 @@ +""" +Integration tests for optional or resource-heavy method end-to-end checks. + +This module is intentionally excluded from default ``unittest`` discovery by +using an ``integration_*.py`` filename. +""" + +import unittest + +import quapy as qp +from quapy.functional import check_prevalence_vector + + +class IntegrationMethodsTest(unittest.TestCase): + + def test_quanet(self): + try: + import quapy.classification.neural + except ModuleNotFoundError: + print('the torch package is not installed; skipping integration test for QuaNet') + return + + qp.environ['SAMPLE_SIZE'] = 10 + + dataset = qp.datasets.fetch_reviews('kindle', pickle=True).reduce() + qp.data.preprocessing.index(dataset, min_df=5, inplace=True) + + from quapy.classification.neural import CNNnet + from quapy.classification.neural import NeuralClassifierTrainer + from quapy.method.meta import QuaNet + + cnn = CNNnet(dataset.vocabulary_size, dataset.n_classes) + learner = NeuralClassifierTrainer(cnn, device='cpu') + model = QuaNet(learner, device='cpu', n_epochs=2, tr_iter_per_poch=10, va_iter_per_poch=10, patience=2) + + model.fit(*dataset.training.Xy) + estim_prevalences = model.predict(dataset.test.instances) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_evaluation.py b/quapy/tests/test_evaluation.py index 05d661a..871b6ec 100644 --- a/quapy/tests/test_evaluation.py +++ b/quapy/tests/test_evaluation.py @@ -1,43 +1,41 @@ +import inspect import unittest - -import numpy as np - -import quapy as qp -from sklearn.linear_model import LogisticRegression from time import time +import numpy as np +from sklearn.linear_model import LogisticRegression + +import quapy as qp from quapy.error import QUANTIFICATION_ERROR_SINGLE_NAMES from quapy.method.aggregative import EMQ, PCC from quapy.method.base import BaseQuantifier +from quapy.tests._synthetic import make_dataset class EvalTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.data = make_dataset(n_train=140, n_test=90, n_classes=2, random_state=7, name='eval') + def test_eval_speedup(self): - """ - Checks whether the speed-up heuristics used by qp.evaluation work, i.e., actually save time - """ - - data = qp.datasets.fetch_reviews('hp', tfidf=True, min_df=10, pickle=True) - train, test = data.training, data.test - - protocol = qp.protocol.APP(test, sample_size=1000, n_prevalences=11, repeats=1, random_state=1) + train, test = self.data.training, self.data.test + protocol = qp.protocol.APP(test, sample_size=30, n_prevalences=5, repeats=1, random_state=1) class SlowLR(LogisticRegression): def predict_proba(self, X): - import time - time.sleep(1) + import time as _time + _time.sleep(0.05) return super().predict_proba(X) - emq = EMQ(SlowLR()).fit(*train.Xy) + emq = EMQ(SlowLR(max_iter=1000)).fit(*train.Xy) tinit = time() - score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', verbose=True, aggr_speedup='force') - tend_optim = time()-tinit - print(f'evaluation (with optimization) took {tend_optim}s [MAE={score:.4f}]') + score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', aggr_speedup='force') + tend_optim = time() - tinit + self.assertTrue(isinstance(score, float)) class NonAggregativeEMQ(BaseQuantifier): - def __init__(self, cls): self.emq = EMQ(cls) @@ -48,31 +46,32 @@ class EvalTestCase(unittest.TestCase): self.emq.fit(X, y) return self - emq = NonAggregativeEMQ(SlowLR()).fit(*train.Xy) + emq = NonAggregativeEMQ(SlowLR(max_iter=1000)).fit(*train.Xy) tinit = time() - score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', verbose=True) + score = qp.evaluation.evaluate(emq, protocol, error_metric='mae') tend_no_optim = time() - tinit - print(f'evaluation (w/o optimization) took {tend_no_optim}s [MAE={score:.4f}]') - - self.assertEqual(tend_no_optim>(tend_optim/2), True) + self.assertTrue(isinstance(score, float)) + self.assertGreater(tend_no_optim, tend_optim) def test_evaluation_output(self): - """ - Checks the evaluation functions return correct types for different error_metrics - """ + train, test = self.data.training, self.data.test + qp.environ['SAMPLE_SIZE'] = 30 + protocol = qp.protocol.APP(test, sample_size=30, n_prevalences=5, repeats=1, random_state=0) + q = PCC(LogisticRegression(max_iter=1000)).fit(*train.Xy) - data = qp.datasets.fetch_reviews('hp', tfidf=True, min_df=10, pickle=True).reduce(n_train=100, n_test=100) - train, test = data.training, data.test + def supports_evaluation(err): + required = [ + p for p in inspect.signature(err).parameters.values() + if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.default is inspect._empty + ] + return len(required) <= 2 - qp.environ['SAMPLE_SIZE']=100 - - protocol = qp.protocol.APP(test, random_state=0) - - q = PCC(LogisticRegression()).fit(*train.Xy) - - single_errors = list(QUANTIFICATION_ERROR_SINGLE_NAMES) - averaged_errors = ['m'+e for e in single_errors] + single_errors = [ + e for e in QUANTIFICATION_ERROR_SINGLE_NAMES + if supports_evaluation(qp.error.from_name(e)) + ] + averaged_errors = ['m' + e for e in single_errors] single_errors = single_errors + [qp.error.from_name(e) for e in single_errors] averaged_errors = averaged_errors + [qp.error.from_name(e) for e in averaged_errors] for error_metric, averaged_error_metric in zip(single_errors, averaged_errors): @@ -81,7 +80,6 @@ class EvalTestCase(unittest.TestCase): scores = qp.evaluation.evaluate(q, protocol, error_metric=error_metric) self.assertTrue(isinstance(scores, np.ndarray)) - self.assertEqual(scores.mean(), score) diff --git a/quapy/tests/test_methods.py b/quapy/tests/test_methods.py index 3e4149e..9c07b33 100644 --- a/quapy/tests/test_methods.py +++ b/quapy/tests/test_methods.py @@ -1,125 +1,122 @@ import itertools +import inspect import unittest from sklearn.linear_model import LogisticRegression -import quapy as qp +from quapy.method import AGGREGATIVE_METHODS, BINARY_METHODS, NON_AGGREGATIVE_METHODS from quapy.method.aggregative import ACC from quapy.method.meta import Ensemble -from quapy.method import AGGREGATIVE_METHODS, BINARY_METHODS, NON_AGGREGATIVE_METHODS from quapy.functional import check_prevalence_vector +from quapy.tests._synthetic import make_dataset +import quapy as qp -# a random selection of composed methods to test the qunfold integration -from quapy.method.composable import check_compatible_qunfold_version +OPTIONAL_AGGREGATIVE_METHODS = { + 'BayesianCC', + 'BayesianKDEy', + 'BayesianMAPLS', + 'PQ', +} -from quapy.method.composable import ( - ComposableQuantifier, - LeastSquaresLoss, - HellingerSurrogateLoss, - ClassRepresentation, - HistogramRepresentation, - CVClassifier -) - -COMPOSABLE_METHODS = [ - ComposableQuantifier( # ACC - LeastSquaresLoss(), - ClassRepresentation(CVClassifier(LogisticRegression())) - ), - ComposableQuantifier( # HDy - HellingerSurrogateLoss(), - HistogramRepresentation( - 3, # 3 bins per class - preprocessor = ClassRepresentation(CVClassifier(LogisticRegression())) - ) - ), -] class TestMethods(unittest.TestCase): - tiny_dataset_multiclass = qp.datasets.fetch_UCIMulticlassDataset('academic-success').reduce(n_test=10) - tiny_dataset_binary = qp.datasets.fetch_UCIBinaryDataset('ionosphere').reduce(n_test=10) + tiny_dataset_multiclass = make_dataset( + n_train=140, n_test=40, n_classes=3, n_features=12, random_state=11, name='synthetic-multiclass' + ) + tiny_dataset_binary = make_dataset( + n_train=140, n_test=40, n_classes=2, n_features=12, random_state=13, name='synthetic-binary' + ) datasets = [tiny_dataset_binary, tiny_dataset_multiclass] def test_aggregative(self): for dataset in TestMethods.datasets: - learner = LogisticRegression() + learner = LogisticRegression(max_iter=2000) learner.fit(*dataset.training.Xy) for model in AGGREGATIVE_METHODS: + if model.__name__ in OPTIONAL_AGGREGATIVE_METHODS: + continue if not dataset.binary and model in BINARY_METHODS: - print(f'skipping the test of binary model {model.__name__} on multiclass dataset {dataset.name}') continue - q = model(learner, fit_classifier=False) - print('testing', q) + kwargs = {'fit_classifier': False} + if 'val_split' in inspect.signature(model.__init__).parameters: + kwargs['val_split'] = None + q = model(learner, **kwargs) q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) self.assertTrue(check_prevalence_vector(estim_prevalences)) def test_non_aggregative(self): for dataset in TestMethods.datasets: - for model in NON_AGGREGATIVE_METHODS: if not dataset.binary and model in BINARY_METHODS: - print(f'skipping the test of binary model {model.__name__} on multiclass dataset {dataset.name}') continue q = model() - print(f'testing {q} on dataset {dataset.name}') q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) self.assertTrue(check_prevalence_vector(estim_prevalences)) def test_ensembles(self): - qp.environ['SAMPLE_SIZE'] = 10 + qp.environ['SAMPLE_SIZE'] = 20 - base_quantifier = ACC(LogisticRegression()) + def policy_supported(policy): + if policy in {'ave', 'ptr', 'ds'}: + return True + err = qp.error.from_name(policy) + required = [ + p for p in inspect.signature(err).parameters.values() + if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.default is inspect._empty + ] + return len(required) <= 2 + + base_quantifier = ACC(LogisticRegression(max_iter=2000)) for dataset, policy in itertools.product(TestMethods.datasets, Ensemble.VALID_POLICIES): + if not policy_supported(policy): + continue if not dataset.binary and policy == 'ds': - print(f'skipping the test of binary policy ds on non-binary dataset {dataset}') continue - print(f'testing {base_quantifier} on dataset {dataset.name} with {policy=}') - ensemble = Ensemble(quantifier=base_quantifier, size=3, policy=policy, n_jobs=-1) + ensemble = Ensemble(quantifier=base_quantifier, size=3, policy=policy, n_jobs=1) ensemble.fit(*dataset.training.Xy) estim_prevalences = ensemble.predict(dataset.test.instances) self.assertTrue(check_prevalence_vector(estim_prevalences)) - def test_quanet(self): + def test_composable(self): try: - import quapy.classification.neural - except ModuleNotFoundError: - print('the torch package is not installed; skipping unit test for QuaNet') + from quapy.method.composable import check_compatible_qunfold_version + from quapy.method.composable import ( + ComposableQuantifier, + LeastSquaresLoss, + HellingerSurrogateLoss, + ClassRepresentation, + HistogramRepresentation, + CVClassifier, + ) + except ImportError: return - qp.environ['SAMPLE_SIZE'] = 10 + composable_methods = [ + ComposableQuantifier( + LeastSquaresLoss(), + ClassRepresentation(CVClassifier(LogisticRegression())) + ), + ComposableQuantifier( + HellingerSurrogateLoss(), + HistogramRepresentation( + 3, + preprocessor=ClassRepresentation(CVClassifier(LogisticRegression())) + ) + ), + ] - # load the kindle dataset as text, and convert words to numerical indexes - dataset = qp.datasets.fetch_reviews('kindle', pickle=True).reduce() - qp.data.preprocessing.index(dataset, min_df=5, inplace=True) - - from quapy.classification.neural import CNNnet - cnn = CNNnet(dataset.vocabulary_size, dataset.n_classes) - - from quapy.classification.neural import NeuralClassifierTrainer - learner = NeuralClassifierTrainer(cnn, device='cpu') - - from quapy.method.meta import QuaNet - model = QuaNet(learner, device='cpu', n_epochs=2, tr_iter_per_poch=10, va_iter_per_poch=10, patience=2) - - model.fit(*dataset.training.Xy) - estim_prevalences = model.predict(dataset.test.instances) - self.assertTrue(check_prevalence_vector(estim_prevalences)) - - def test_composable(self): if check_compatible_qunfold_version(): for dataset in TestMethods.datasets: - for q in COMPOSABLE_METHODS: - print('testing', q) + for q in composable_methods: q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) - print(estim_prevalences) self.assertTrue(check_prevalence_vector(estim_prevalences)) else: from quapy.method.composable import __old_version_message diff --git a/quapy/tests/test_modsel.py b/quapy/tests/test_modsel.py index 6423b4e..0f3c8f6 100644 --- a/quapy/tests/test_modsel.py +++ b/quapy/tests/test_modsel.py @@ -1,104 +1,87 @@ +import time import unittest import numpy as np from sklearn.linear_model import LogisticRegression -import quapy as qp from quapy.method.aggregative import PACC from quapy.model_selection import GridSearchQ from quapy.protocol import APP -import time +from quapy.tests._synthetic import make_dataset class ModselTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + data = make_dataset( + n_train=220, + n_test=120, + n_classes=2, + n_features=16, + class_sep=1.8, + random_state=1, + name='modsel', + ) + cls.training, cls.validation = data.training.split_stratified(0.7, random_state=1) + def test_modsel(self): """ - Checks whether a model selection exploration takes a good hyperparameter + Checks whether a model selection exploration picks the better hyperparameter. """ - q = PACC(LogisticRegression(random_state=1, max_iter=5000)) - - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=10).reduce(random_state=1) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': [0.000001, 10.]} - app = APP(validation, sample_size=100, random_state=1) + param_grid = {'classifier__C': [0.000001, 10.0]} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) q = GridSearchQ( - q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, verbose=True, n_jobs=-1 - ).fit(*training.Xy) - print('best params', q.best_params_) - print('best score', q.best_score_) + q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, verbose=False, n_jobs=-1 + ).fit(*self.training.Xy) self.assertEqual(q.best_params_['classifier__C'], 10.0) self.assertEqual(q.best_model().get_params()['classifier__C'], 10.0) def test_modsel_parallel(self): """ - Checks whether a parallelized model selection actually is faster than a sequential exploration but - obtains the same optimal parameters + Checks whether sequential and parallel model selection agree on the best parameters. """ - q = PACC(LogisticRegression(random_state=1, max_iter=3000)) - - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=50) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': np.logspace(-3,3,7), 'classifier__class_weight': ['balanced', None]} - app = APP(validation, sample_size=100, random_state=1) + param_grid = {'classifier__C': np.logspace(-3, 3, 7), 'classifier__class_weight': ['balanced', None]} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) def do_gridsearch(n_jobs): - print('starting model selection in sequential exploration') t_init = time.time() modsel = GridSearchQ( - q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, n_jobs=n_jobs, verbose=True - ).fit(*training.Xy) - t_end = time.time()-t_init - best_c = modsel.best_params_['classifier__C'] - print(f'[done] took {t_end:.2f}s best C = {best_c}') - return t_end, best_c + q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, n_jobs=n_jobs, verbose=False + ).fit(*self.training.Xy) + t_end = time.time() - t_init + return t_end, modsel.best_params_ - tend_seq, best_c_seq = do_gridsearch(n_jobs=1) - tend_par, best_c_par = do_gridsearch(n_jobs=-1) - - print(tend_seq, best_c_seq) - print(tend_par, best_c_par) - - self.assertEqual(best_c_seq, best_c_par) - self.assertLess(tend_par, tend_seq) + _, best_seq = do_gridsearch(n_jobs=1) + _, best_par = do_gridsearch(n_jobs=-1) + self.assertEqual(best_seq, best_par) def test_modsel_timeout(self): class SlowLR(LogisticRegression): def fit(self, X, y, sample_weight=None): - import time - time.sleep(10) - super(SlowLR, self).fit(X, y, sample_weight) + time.sleep(2) + return super().fit(X, y, sample_weight) - q = PACC(SlowLR()) + q = PACC(SlowLR(max_iter=1000)) + param_grid = {'classifier__C': np.logspace(-1, 1, 3)} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=10).reduce(random_state=1) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': np.logspace(-1,1,3)} - app = APP(validation, sample_size=100, random_state=1) - - print('Expecting TimeoutError to be raised') modsel = GridSearchQ( - q, param_grid, protocol=app, timeout=3, n_jobs=-1, verbose=True, raise_errors=True + q, param_grid, protocol=app, timeout=1, n_jobs=-1, verbose=False, raise_errors=True ) with self.assertRaises(TimeoutError): - modsel.fit(*training.Xy) + modsel.fit(*self.training.Xy) - print('Expecting ValueError to be raised') modsel = GridSearchQ( - q, param_grid, protocol=app, timeout=3, n_jobs=-1, verbose=True, raise_errors=False + q, param_grid, protocol=app, timeout=1, n_jobs=-1, verbose=False, raise_errors=False ) with self.assertRaises(ValueError): - # this exception is not raised because of the timeout, but because no combination of hyperparams - # succedded (in this case, a ValueError is raised, regardless of "raise_errors" - modsel.fit(*training.Xy) + modsel.fit(*self.training.Xy) if __name__ == '__main__': diff --git a/quapy/tests/test_protocols.py b/quapy/tests/test_protocols.py index 4850bd4..329762a 100644 --- a/quapy/tests/test_protocols.py +++ b/quapy/tests/test_protocols.py @@ -3,7 +3,7 @@ import numpy as np import quapy.functional from quapy.data import LabelledCollection -from quapy.protocol import APP, NPP, UPP, DomainMixer, AbstractStochasticSeededProtocol +from quapy.protocol import APP, NPP, UPP, DomainMixer, AbstractStochasticSeededProtocol, DirichletProtocol def mock_labelled_collection(prefix=''): @@ -138,6 +138,31 @@ class TestProtocols(unittest.TestCase): self.assertNotEqual(samples1, samples2) + def test_dirichlet_replicate(self): + data = mock_labelled_collection() + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=42) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertEqual(samples1, samples2) + + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=0) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertEqual(samples1, samples2) + + def test_dirichlet_not_replicate(self): + data = mock_labelled_collection() + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=None) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertNotEqual(samples1, samples2) + def test_covariate_shift_replicate(self): dataA = mock_labelled_collection('domA') dataB = mock_labelled_collection('domB') diff --git a/quapy/tests/test_replicability.py b/quapy/tests/test_replicability.py index a174992..b35c4ab 100644 --- a/quapy/tests/test_replicability.py +++ b/quapy/tests/test_replicability.py @@ -1,19 +1,29 @@ import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + import quapy as qp +import quapy.functional as F from quapy.data import LabelledCollection from quapy.functional import strprev -from sklearn.linear_model import LogisticRegression -import numpy as np from quapy.method.aggregative import PACC -import quapy.functional as F +from quapy.tests._synthetic import make_dataset class TestReplicability(unittest.TestCase): - def test_prediction_replicability(self): + @classmethod + def setUpClass(cls): + cls.binary_dataset = make_dataset( + n_train=180, n_test=80, n_classes=2, n_features=10, random_state=21, name='rep-binary' + ) + cls.multiclass_dataset = make_dataset( + n_train=180, n_test=80, n_classes=3, n_features=12, random_state=23, name='rep-multiclass' + ) - dataset = qp.datasets.fetch_UCIBinaryDataset('yeast') - train, test = dataset.train_test + def test_prediction_replicability(self): + train, test = self.binary_dataset.train_test with qp.util.temp_seed(0): lr = LogisticRegression(random_state=0, max_iter=10000) @@ -29,7 +39,6 @@ class TestReplicability(unittest.TestCase): self.assertEqual(str_prev1, str_prev2) - def test_samping_replicability(self): def equal_collections(c1, c2, value=True): @@ -60,53 +69,33 @@ class TestReplicability(unittest.TestCase): sample2 = data.sampling(50, *[0.7, 0.3]) equal_collections(sample1, sample2, True) - sample1 = data.sampling(50, *[0.7, 0.3], random_state=0) - sample2 = data.sampling(50, *[0.7, 0.3], random_state=0) - equal_collections(sample1, sample2, True) - sample1_tr, sample1_te = data.split_stratified(train_prop=0.7, random_state=0) sample2_tr, sample2_te = data.split_stratified(train_prop=0.7, random_state=0) equal_collections(sample1_tr, sample2_tr, True) equal_collections(sample1_te, sample2_te, True) - with qp.util.temp_seed(0): - sample1_tr, sample1_te = data.split_stratified(train_prop=0.7) - with qp.util.temp_seed(0): - sample2_tr, sample2_te = data.split_stratified(train_prop=0.7) - equal_collections(sample1_tr, sample2_tr, True) - equal_collections(sample1_te, sample2_te, True) - - def test_parallel_replicability(self): - - train, test = qp.datasets.fetch_UCIMulticlassDataset('dry-bean').reduce().train_test - - test = test.sampling(500, *[0.1, 0.0, 0.1, 0.1, 0.2, 0.5, 0.0]) + train, test = self.multiclass_dataset.train_test + test = test.sampling(60, *[0.2, 0.3, 0.5], random_state=4) with qp.util.temp_seed(10): - pacc = PACC(LogisticRegression(), val_split=.5, n_jobs=2) + pacc = PACC(LogisticRegression(max_iter=5000), val_split=.5, n_jobs=2) pacc.fit(*train.Xy) prev1 = F.strprev(pacc.predict(test.instances)) with qp.util.temp_seed(0): - pacc = PACC(LogisticRegression(), val_split=.5, n_jobs=2) + pacc = PACC(LogisticRegression(max_iter=5000), val_split=.5, n_jobs=2) pacc.fit(*train.Xy) prev2 = F.strprev(pacc.predict(test.instances)) with qp.util.temp_seed(0): - pacc = PACC(LogisticRegression(), val_split=.5, n_jobs=2) + pacc = PACC(LogisticRegression(max_iter=5000), val_split=.5, n_jobs=2) pacc.fit(*train.Xy) prev3 = F.strprev(pacc.predict(test.instances)) - print(prev1) - print(prev2) - print(prev3) - self.assertNotEqual(prev1, prev2) self.assertEqual(prev2, prev3) - - if __name__ == '__main__': unittest.main() From a1dff57db064fa2e75bda14efb1e0b71da229fd4 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Tue, 30 Jun 2026 18:02:23 +0200 Subject: [PATCH 26/43] refactoring and cleaning up --- CHANGE_LOG.txt | 5 +- docs/build/html/_modules/index.html | 466 ++++- .../quapy/classification/calibration.html | 212 ++- .../quapy/classification/methods.html | 85 +- .../quapy/classification/svmperf.html | 81 +- docs/build/html/_modules/quapy/data/base.html | 201 ++- .../html/_modules/quapy/data/datasets.html | 1074 +++++++----- .../_modules/quapy/data/preprocessing.html | 134 +- .../html/_modules/quapy/data/reader.html | 48 +- docs/build/html/_modules/quapy/error.html | 377 +++- .../build/html/_modules/quapy/evaluation.html | 75 +- .../build/html/_modules/quapy/functional.html | 953 +++++++--- .../html/_modules/quapy/method/_kdey.html | 344 ++-- .../quapy/method/_threshold_optim.html | 295 ++-- .../_modules/quapy/method/aggregative.html | 1549 +++++++++++------ .../html/_modules/quapy/method/base.html | 158 +- .../html/_modules/quapy/method/meta.html | 456 +++-- .../quapy/method/non_aggregative.html | 298 +++- .../html/_modules/quapy/model_selection.html | 211 ++- docs/build/html/_modules/quapy/plot.html | 261 ++- docs/build/html/_modules/quapy/protocol.html | 381 ++-- docs/build/html/_modules/quapy/util.html | 213 ++- .../_sphinx_javascript_frameworks_compat.js | 17 +- docs/build/html/_static/css/badge_only.css | 2 +- docs/build/html/_static/css/theme.css | 2 +- docs/build/html/_static/sphinx_highlight.js | 81 +- docs/source/conf.py | 3 +- quapy/error.py | 9 +- quapy/functional.py | 31 +- quapy/method/__init__.py | 3 +- quapy/method/_kdey.py | 7 +- quapy/method/aggregative.py | 155 +- quapy/method/non_aggregative.py | 8 +- quapy/tests/test_methods.py | 54 +- 34 files changed, 5812 insertions(+), 2437 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index 96cf26a..06fb255 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -8,9 +8,10 @@ Change Log 0.2.1 - Extended KDEy with Aitchison/ILR kernels, shrinkage, and improved numerical stability. - Added TemperatureScalingFromLogits for calibrating pretrained logits. - Added DirichletProtocol for prevalence sampling from Dirichlet priors. -- Added ReadMe method by Daniel Hopkins and Gary King +- Added ReadMe method by Daniel Hopkins and Gary King. - Internal index in LabelledCollection is now "lazy", and is only constructed if required. -- Improved unit testing and separated integration tests +- Improved unit testing and separated integration tests. +- Added RLLS (Regularized Learning for Domain Adaptation under Label Shifts) method. Change Log 0.2.0 ----------------- diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html index 9951ba2..ab28d32 100644 --- a/docs/build/html/_modules/index.html +++ b/docs/build/html/_modules/index.html @@ -1,78 +1,335 @@ + - - - - - Overview: module code — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + Overview: module code — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

All modules for which code is available

-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
+
+ +
+ +
-
-
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.19.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/classification/calibration.html b/docs/build/html/_modules/quapy/classification/calibration.html index 6576c6c..1eac806 100644 --- a/docs/build/html/_modules/quapy/classification/calibration.html +++ b/docs/build/html/_modules/quapy/classification/calibration.html @@ -1,23 +1,20 @@ + + - + - quapy.classification.calibration — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.classification.calibration — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@ @@ -71,12 +74,13 @@

Source code for quapy.classification.calibration

-from copy import deepcopy
+from copy import deepcopy
 
-from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
-from sklearn.base import BaseEstimator, clone
-from sklearn.model_selection import cross_val_predict, train_test_split
-import numpy as np
+from sklearn.base import BaseEstimator, clone
+from sklearn.model_selection import cross_val_predict, train_test_split
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils.validation import check_X_y
+import numpy as np
 
 
 # Wrappers of calibration defined by Alexandari et al. in paper <http://proceedings.mlr.press/v119/alexandari20a.html>
@@ -84,7 +88,20 @@
 # see https://github.com/kundajelab/abstention
 
 
-
[docs]class RecalibratedProbabilisticClassifier:
+def _require_abstention_calibration():
+    try:
+        from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
+    except ImportError as exc:
+        raise ImportError(
+            "Calibration methods in quapy.classification.calibration require the optional "
+            "'abstention' package."
+        ) from exc
+    return NoBiasVectorScaling, TempScaling, VectorScaling
+
+
+

+[docs]
+class RecalibratedProbabilisticClassifier:
     """
     Abstract class for (re)calibration method from `abstention.calibration`, as defined in
     `Alexandari, A., Kundaje, A., & Shrikumar, A. (2020, November). Maximum likelihood with bias-corrected calibration
@@ -94,7 +111,10 @@
     pass

 
 
-
[docs]class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
+
+

+[docs]
+class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
     """
     Applies a (re)calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_.
@@ -110,14 +130,16 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
         self.classifier = classifier
         self.calibrator = calibrator
         self.val_split = val_split
         self.n_jobs = n_jobs
         self.verbose = verbose
 
-
[docs]    def fit(self, X, y):
+

+[docs]
+    def fit(self, X, y):
         """
         Fits the calibration for the probabilistic classifier.
 
@@ -135,7 +157,10 @@
                 raise ValueError('wrong value for val_split: the proportion of validation documents must be in (0,1)')
             return self.fit_tr_val(X, y)

 
-
[docs]    def fit_cv(self, X, y):
+
+

+[docs]
+    def fit_cv(self, X, y):
         """
         Fits the calibration in a cross-validation manner, i.e., it generates posterior probabilities for all
         training instances via cross-validation, and then retrains the classifier on all training instances.
@@ -153,7 +178,10 @@
         self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[y], posterior_supplied=True)
         return self

 
-
[docs]    def fit_tr_val(self, X, y):
+
+

+[docs]
+    def fit_tr_val(self, X, y):
         """
         Fits the calibration in a train/val-split manner, i.e.t, it partitions the training instances into a
         training and a validation set, and then uses the training samples to learn classifier which is then used
@@ -171,7 +199,10 @@
         self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[yva], posterior_supplied=True)
         return self

 
-
[docs]    def predict(self, X):
+
+

+[docs]
+    def predict(self, X):
         """
         Predicts class labels for the data instances in `X`
 
@@ -180,7 +211,10 @@
         """
         return self.classifier.predict(X)

 
-
[docs]    def predict_proba(self, X):
+
+

+[docs]
+    def predict_proba(self, X):
         """
         Generates posterior probabilities for the data instances in `X`
 
@@ -190,8 +224,9 @@
         posteriors = self.classifier.predict_proba(X)
         return self.calibration_function(posteriors)

 
+
     @property
-    def classes_(self):
+    def classes_(self):
         """
         Returns the classes on which the classifier has been trained on
 
@@ -200,7 +235,10 @@
         return self.classifier.classes_

 
 
-
[docs]class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the No-Bias Vector Scaling (NBVS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -214,7 +252,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        NoBiasVectorScaling, _, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = NoBiasVectorScaling(verbose=verbose)
         self.val_split = val_split
@@ -222,7 +261,10 @@
         self.verbose = verbose

 
 
-
[docs]class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Bias-Corrected Temperature Scaling (BCTS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -236,7 +278,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, TempScaling, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = TempScaling(verbose=verbose, bias_positions='all')
         self.val_split = val_split
@@ -244,7 +287,10 @@
         self.verbose = verbose

 
 
-
[docs]class TSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class TSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Temperature Scaling (TS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -258,7 +304,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, TempScaling, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = TempScaling(verbose=verbose)
         self.val_split = val_split
@@ -266,7 +313,10 @@
         self.verbose = verbose

 
 
-
[docs]class VSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class VSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Vector Scaling (VS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -280,13 +330,101 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, _, VectorScaling = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = VectorScaling(verbose=verbose)
         self.val_split = val_split
         self.n_jobs = n_jobs
         self.verbose = verbose

 
+
+
+

+[docs]
+class TemperatureScalingFromLogits(BaseEstimator):
+    """
+    Calibrates a matrix of logits by learning a temperature-scaling mapping
+    with the calibration methods from `abstention.calibration`.
+
+    This estimator is useful when the inputs are already logits produced by a
+    pretrained classifier, and the goal is to transform them directly into
+    calibrated posterior probabilities without retraining the underlying model.
+
+    :param bias_corrected: if True, uses Bias-Corrected Temperature Scaling
+        (BCTS); otherwise, uses standard Temperature Scaling (TS)
+    :param verbose: whether the underlying calibrator should display progress
+        information
+    """
+
+    def __init__(self, bias_corrected=False, verbose=False):
+        self.bias_corrected = bias_corrected
+        self.verbose = verbose
+
+

+[docs]
+    def fit(self, X, y):
+        """
+        Fits the logits calibrator.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :param y: array-like of shape `(n_samples,)` containing class labels
+        :return: self
+        """
+        X, y = check_X_y(X, y)
+
+        self.label_encoder_ = LabelEncoder()
+        y_enc = self.label_encoder_.fit_transform(y)
+        self.classes_ = self.label_encoder_.classes_
+
+        n_classes = len(self.classes_)
+        logits_dim = X.shape[1]
+        if n_classes != logits_dim:
+            raise ValueError(
+                f'mismatch between the number of classes ({n_classes}) and the '
+                f'dimensionality of the logits ({logits_dim})'
+            )
+
+        _, TempScaling, _ = _require_abstention_calibration()
+        calibrator = TempScaling(
+            verbose=self.verbose,
+            bias_positions='all' if self.bias_corrected else [],
+        )
+        self.calibrator_ = calibrator
+        self.calibration_function_ = calibrator(X, np.eye(n_classes)[y_enc])
+        return self

+
+
+

+[docs]
+    def predict_proba(self, X):
+        """
+        Converts logits into calibrated posterior probabilities.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :return: array-like of shape `(n_samples, n_classes)` with calibrated
+            posterior probabilities
+        """
+        return self.calibration_function_(X)

+
+
+

+[docs]
+    def predict(self, X):
+        """
+        Predicts class labels after calibration.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :return: array-like of shape `(n_samples,)` with class label
+            predictions
+        """
+        posteriors = self.predict_proba(X)
+        return self.label_encoder_.inverse_transform(np.argmax(posteriors, axis=1))

+

+
diff --git a/docs/build/html/_modules/quapy/classification/methods.html b/docs/build/html/_modules/quapy/classification/methods.html index 883f802..d921a91 100644 --- a/docs/build/html/_modules/quapy/classification/methods.html +++ b/docs/build/html/_modules/quapy/classification/methods.html @@ -1,22 +1,20 @@ + + - quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - + + + + + @@ -42,7 +40,13 @@ @@ -70,14 +74,15 @@

Source code for quapy.classification.methods

-from sklearn.base import BaseEstimator
-from sklearn.decomposition import TruncatedSVD
-from sklearn.linear_model import LogisticRegression
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.decomposition import TruncatedSVD
+from sklearn.linear_model import LogisticRegression
 
 
 

 [docs]
-class LowRankLogisticRegression(BaseEstimator):
+class LowRankLogisticRegression(BaseEstimator):
     """
     An example of a classification method (i.e., an object that implements `fit`, `predict`, and `predict_proba`)
     that also generates embedded inputs (i.e., that implements `transform`), as those required for
@@ -91,13 +96,13 @@
         `Logistic Regression <https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html>`__ classifier
     """
 
-    def __init__(self, n_components=100, **kwargs):
+    def __init__(self, n_components=100, **kwargs):
         self.n_components = n_components
         self.classifier = LogisticRegression(**kwargs)
 
 

 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator.
 
@@ -110,7 +115,7 @@
 
 

 [docs]
-    def set_params(self, **params):
+    def set_params(self, **params):
         """
         Set the parameters of this estimator.
 
@@ -127,7 +132,7 @@
 
 

 [docs]
-    def fit(self, X, y):
+    def fit(self, X, y):
         """
         Fit the model according to the given training data. The fit consists of
         fitting `TruncatedSVD` and then `LogisticRegression` on the low-rank representation.
@@ -148,7 +153,7 @@
 
 

 [docs]
-    def predict(self, X):
+    def predict(self, X):
         """
         Predicts labels for the instances `X` embedded into the low-rank space.
 
@@ -162,7 +167,7 @@
 
 

 [docs]
-    def predict_proba(self, X):
+    def predict_proba(self, X):
         """
         Predicts posterior probabilities for the instances `X` embedded into the low-rank space.
 
@@ -175,7 +180,7 @@
 
 

 [docs]
-    def transform(self, X):
+    def transform(self, X):
         """
         Returns the low-rank approximation of `X` with `n_components` dimensions, or `X` unaltered if
         `n_components` >= `X.shape[1]`.
@@ -188,6 +193,38 @@
         return self.pca.transform(X)

 

 
+
+
+

+[docs]
+class MockClassifierFromPosteriors(BaseEstimator):
+    """
+    Mock classifier that bypasses classifier training when the input instances
+    are already posterior probabilities produced by a pretrained probabilistic
+    classifier.
+
+    :param X: arrays of shape `(n_samples, n_classes)` are interpreted as posterior probabilities
+    """
+
+

+[docs]
+    def fit(self, X, y):
+        self.classes_ = np.sort(np.unique(y))
+        return self

+
+
+

+[docs]
+    def predict(self, X):
+        return np.argmax(X, axis=1)

+
+
+

+[docs]
+    def predict_proba(self, X):
+        return X

+

+
diff --git a/docs/build/html/_modules/quapy/classification/svmperf.html b/docs/build/html/_modules/quapy/classification/svmperf.html index 959ad48..9b60c1a 100644 --- a/docs/build/html/_modules/quapy/classification/svmperf.html +++ b/docs/build/html/_modules/quapy/classification/svmperf.html @@ -1,22 +1,20 @@ + + - quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - + + + + + @@ -42,7 +40,13 @@ @@ -70,21 +74,21 @@

Source code for quapy.classification.svmperf

-import random
-import shutil
-import subprocess
-import tempfile
-from os import remove, makedirs
-from os.path import join, exists
-from subprocess import PIPE, STDOUT
-import numpy as np
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.datasets import dump_svmlight_file
+import random
+import shutil
+import subprocess
+import tempfile
+from os import remove, makedirs
+from os.path import join, exists
+from subprocess import PIPE, STDOUT
+import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.datasets import dump_svmlight_file
 
 
 

 [docs]
-class SVMperf(BaseEstimator, ClassifierMixin):
+class SVMperf(BaseEstimator, ClassifierMixin):
     """A wrapper for the `SVM-perf package <https://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html>`__ by Thorsten Joachims.
     When using losses for quantification, the source code has to be patched. See
     the `installation documentation <https://hlt-isti.github.io/QuaPy/build/html/Installation.html#svm-perf-with-quantification-oriented-losses>`__
@@ -106,31 +110,20 @@
     # losses with their respective codes in svm_perf implementation
     valid_losses = {'01':0, 'f1':1, 'kld':12, 'nkld':13, 'q':22, 'qacc':23, 'qf1':24, 'qgm':25, 'mae':26, 'mrae':27}
 
-    def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
-        assert exists(svmperf_base), f'path {svmperf_base} does not seem to point to a valid path'
+    def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
+        assert exists(svmperf_base), \
+            (f'path {svmperf_base} does not seem to point to a valid path;'
+             f'did you install svm-perf? '
+             f'see instructions in https://hlt-isti.github.io/QuaPy/manuals/explicit-loss-minimization.html')
         self.svmperf_base = svmperf_base
         self.C = C
         self.verbose = verbose
         self.loss = loss
         self.host_folder = host_folder
 
-    # def set_params(self, **parameters):
-    #     """
-    #     Set the hyper-parameters for svm-perf. Currently, only the `C` and `loss` parameters are supported
-    #
-    #     :param parameters: a `**kwargs` dictionary `{'C': <float>}`
-    #     """
-    #     assert sorted(list(parameters.keys())) == ['C', 'loss'], \
-    #         'currently, only the C and loss parameters are supported'
-    #     self.C = parameters.get('C', self.C)
-    #     self.loss = parameters.get('loss', self.loss)
-    #
-    # def get_params(self, deep=True):
-    #     return {'C': self.C, 'loss': self.loss}
-
 

 [docs]
-    def fit(self, X, y):
+    def fit(self, X, y):
         """
         Trains the SVM for the multivariate performance loss
 
@@ -180,7 +173,7 @@
 
 

 [docs]
-    def predict(self, X):
+    def predict(self, X):
         """
         Predicts labels for the instances `X`
 
@@ -195,7 +188,7 @@
 
 

 [docs]
-    def decision_function(self, X, y=None):
+    def decision_function(self, X, y=None):
         """
         Evaluate the decision function for the samples in `X`.
 
@@ -231,7 +224,7 @@
         return scores

 
 
-    def __del__(self):
+    def __del__(self):
         if hasattr(self, 'tmpdir'):
             shutil.rmtree(self.tmpdir, ignore_errors=True)

 
diff --git a/docs/build/html/_modules/quapy/data/base.html b/docs/build/html/_modules/quapy/data/base.html
index e3a2e89..27f34de 100644
--- a/docs/build/html/_modules/quapy/data/base.html
+++ b/docs/build/html/_modules/quapy/data/base.html
@@ -1,22 +1,20 @@
+
+
 
 
 
   
   
-  quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -42,7 +40,13 @@
 

         

@@ -70,22 +74,23 @@
            

              
   
Source code for quapy.data.base
-import itertools
-from functools import cached_property
-from typing import Iterable
+import itertools
+from functools import cached_property
+from typing import Iterable
 
-import numpy as np
-from scipy.sparse import issparse
-from scipy.sparse import vstack
-from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
-from numpy.random import RandomState
-from quapy.functional import strprev
-from quapy.util import temp_seed
+import numpy as np
+from scipy.sparse import issparse
+from scipy.sparse import vstack
+from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
+from numpy.random import RandomState
+from quapy.functional import strprev
+from quapy.util import temp_seed
+import quapy.functional as F
 
 
 

 [docs]
-class LabelledCollection:
+class LabelledCollection:
     """
     A LabelledCollection is a set of objects each with a label attached to each of them. 
     This class implements several sampling routines and other utilities.
@@ -97,7 +102,7 @@
         (i.e., a prevalence of 0)
     """
 
-    def __init__(self, instances, labels, classes=None):
+    def __init__(self, instances, labels, classes=None):
         if issparse(instances):
             self.instances = instances
         elif isinstance(instances, list) and len(instances) > 0 and isinstance(instances[0], str):
@@ -106,21 +111,25 @@
         else:
             self.instances = np.asarray(instances)
         self.labels = np.asarray(labels)
-        n_docs = len(self)
         if classes is None:
-            self.classes_ = np.unique(self.labels)
-            self.classes_.sort()
+            self.classes_ = F.classes_from_labels(self.labels)
         else:
             self.classes_ = np.unique(np.asarray(classes))
             self.classes_.sort()
             if len(set(self.labels).difference(set(classes))) > 0:
                 raise ValueError(f'labels ({set(self.labels)}) contain values not included in classes_ ({set(classes)})')
-        self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_}
+        self._index = None
+
+    @property
+    def index(self):
+        if not hasattr(self, '_index') or self._index is None:
+            self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_}
+        return self._index
 
 

 [docs]
     @classmethod
-    def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
+    def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
         """
         Loads a labelled set of data and convert it into a :class:`LabelledCollection` instance. The function in charge
         of reading the instances must be specified. This function can be a custom one, or any of the reading functions
@@ -137,7 +146,7 @@
         return LabelledCollection(*loader_func(path, **loader_kwargs), classes)

 
 
-    def __len__(self):
+    def __len__(self):
         """
         Returns the length of this collection (number of labelled instances)
 
@@ -147,7 +156,7 @@
 
 

 [docs]
-    def prevalence(self):
+    def prevalence(self):
         """
         Returns the prevalence, or relative frequency, of the classes in the codeframe.
 
@@ -159,7 +168,7 @@
 
 

 [docs]
-    def counts(self):
+    def counts(self):
         """
         Returns the number of instances for each of the classes in the codeframe.
 
@@ -170,7 +179,7 @@
 
 
     @property
-    def n_classes(self):
+    def n_classes(self):
         """
         The number of classes
 
@@ -179,7 +188,16 @@
         return len(self.classes_)
 
     @property
-    def binary(self):
+    def n_instances(self):
+        """
+        The number of instances
+
+        :return: integer
+        """
+        return len(self.labels)
+
+    @property
+    def binary(self):
         """
         Returns True if the number of classes is 2
 
@@ -189,12 +207,11 @@
 
 

 [docs]
-    def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
+    def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
         """
         Returns an index to be used to extract a random sample of desired size and desired prevalence values. If the
         prevalence values are not specified, then returns the index of a uniform sampling.
-        For each class, the sampling is drawn with replacement if the requested prevalence is larger than
-        the actual prevalence of the class, or without replacement otherwise.
+        For each class, the sampling is drawn with replacement.
 
         :param size: integer, the requested size
         :param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -209,7 +226,7 @@
         if len(prevs) == self.n_classes - 1:
             prevs = prevs + (1 - sum(prevs),)
         assert len(prevs) == self.n_classes, 'unexpected number of prevalences'
-        assert sum(prevs) == 1, f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
+        assert np.isclose(sum(prevs), 1), f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
 
         # Decide how many instances should be taken for each class in order to satisfy the requested prevalence
         # accurately, and the number of instances in the sample (exactly). If int(size * prevs[i]) (which is
@@ -238,7 +255,7 @@
             for class_, n_requested in n_requests.items():
                 n_candidates = len(self.index[class_])
                 index_sample = self.index[class_][
-                    np.random.choice(n_candidates, size=n_requested, replace=(n_requested > n_candidates))
+                    np.random.choice(n_candidates, size=n_requested, replace=True)
                 ] if n_requested > 0 else []
 
                 indexes_sample.append(index_sample)
@@ -253,11 +270,10 @@
 
 

 [docs]
-    def uniform_sampling_index(self, size, random_state=None):
+    def uniform_sampling_index(self, size, random_state=None):
         """
         Returns an index to be used to extract a uniform sample of desired size. The sampling is drawn
-        with replacement if the requested size is greater than the number of instances, or without replacement
-        otherwise.
+        with replacement.
 
         :param size: integer, the size of the uniform sample
         :param random_state: if specified, guarantees reproducibility of the split.
@@ -267,16 +283,15 @@
             ng = RandomState(seed=random_state)
         else:
             ng = np.random
-        return ng.choice(len(self), size, replace=size > len(self))

+        return ng.choice(len(self), size, replace=True)

 
 
 

 [docs]
-    def sampling(self, size, *prevs, shuffle=True, random_state=None):
+    def sampling(self, size, *prevs, shuffle=True, random_state=None):
         """
         Return a random sample (an instance of :class:`LabelledCollection`) of desired size and desired prevalence
-        values. For each class, the sampling is drawn without replacement if the requested prevalence is larger than
-        the actual prevalence of the class, or with replacement otherwise.
+        values. For each class, the sampling is drawn with replacement.
 
         :param size: integer, the requested size
         :param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -293,11 +308,10 @@
 
 

 [docs]
-    def uniform_sampling(self, size, random_state=None):
+    def uniform_sampling(self, size, random_state=None):
         """
         Returns a uniform sample (an instance of :class:`LabelledCollection`) of desired size. The sampling is drawn
-        with replacement if the requested size is greater than the number of instances, or without replacement
-        otherwise.
+        with replacement.
 
         :param size: integer, the requested size
         :param random_state: if specified, guarantees reproducibility of the split.
@@ -309,7 +323,7 @@
 
 

 [docs]
-    def sampling_from_index(self, index):
+    def sampling_from_index(self, index):
         """
         Returns an instance of :class:`LabelledCollection` whose elements are sampled from this collection using the
         index.
@@ -324,7 +338,7 @@
 
 

 [docs]
-    def split_stratified(self, train_prop=0.6, random_state=None):
+    def split_stratified(self, train_prop=0.6, random_state=None):
         """
         Returns two instances of :class:`LabelledCollection` split with stratification from this collection, at desired
         proportion.
@@ -336,17 +350,17 @@
         :return: two instances of :class:`LabelledCollection`, the first one with `train_prop` elements, and the
             second one with `1-train_prop` elements
         """
-        tr_docs, te_docs, tr_labels, te_labels = train_test_split(
+        tr_X, te_X, tr_y, te_y = train_test_split(
             self.instances, self.labels, train_size=train_prop, stratify=self.labels, random_state=random_state
         )
-        training = LabelledCollection(tr_docs, tr_labels, classes=self.classes_)
-        test = LabelledCollection(te_docs, te_labels, classes=self.classes_)
+        training = LabelledCollection(tr_X, tr_y, classes=self.classes_)
+        test = LabelledCollection(te_X, te_y, classes=self.classes_)
         return training, test

 
 
 

 [docs]
-    def split_random(self, train_prop=0.6, random_state=None):
+    def split_random(self, train_prop=0.6, random_state=None):
         """
         Returns two instances of :class:`LabelledCollection` split randomly from this collection, at desired
         proportion.
@@ -373,7 +387,7 @@
         return training, test

 
 
-    def __add__(self, other):
+    def __add__(self, other):
         """
         Returns a new :class:`LabelledCollection` as the union of this collection with another collection.
         Both labelled collections must have the same classes.
@@ -389,7 +403,7 @@
 

 [docs]
     @classmethod
-    def join(cls, *args: Iterable['LabelledCollection']):
+    def join(cls, *args: Iterable['LabelledCollection']):
         """
         Returns a new :class:`LabelledCollection` as the union of the collections given in input.
 
@@ -430,7 +444,16 @@
 
 
     @property
-    def Xy(self):
+    def classes(self):
+        """
+        Gets an array-like with the classes used in this collection
+
+        :return: array-like
+        """
+        return self.classes_
+
+    @property
+    def Xy(self):
         """
         Gets the instances and labels. This is useful when working with `sklearn` estimators, e.g.:
 
@@ -441,7 +464,7 @@
         return self.instances, self.labels
 
     @property
-    def Xp(self):
+    def Xp(self):
         """
         Gets the instances and the true prevalence. This is useful when implementing evaluation protocols from
         a :class:`LabelledCollection` object.
@@ -451,7 +474,7 @@
         return self.instances, self.prevalence()
 
     @property
-    def X(self):
+    def X(self):
         """
         An alias to self.instances
 
@@ -460,7 +483,7 @@
         return self.instances
 
     @property
-    def y(self):
+    def y(self):
         """
         An alias to self.labels
 
@@ -469,7 +492,7 @@
         return self.labels
 
     @property
-    def p(self):
+    def p(self):
         """
         An alias to self.prevalence()
 
@@ -480,7 +503,7 @@
 
 

 [docs]
-    def stats(self, show=True):
+    def stats(self, show=True):
         """
         Returns (and eventually prints) a dictionary with some stats of this collection. E.g.,:
 
@@ -515,7 +538,7 @@
 
 

 [docs]
-    def kFCV(self, nfolds=5, nrepeats=1, random_state=None):
+    def kFCV(self, nfolds=5, nrepeats=1, random_state=None):
         """
         Generator of stratified folds to be used in k-fold cross validation.
 
@@ -529,13 +552,18 @@
             train = self.sampling_from_index(train_index)
             test = self.sampling_from_index(test_index)
             yield train, test

-

+
+
+    def __repr__(self):
+        repr=f'<{self.n_instances} instances (dtype={type(self.instances[0])}), '
+        repr+=f'n_classes={self.n_classes} {self.classes_}, prevalence={F.strprev(self.prevalence())}>'
+        return repr

 
 
 
 

 [docs]
-class Dataset:
+class Dataset:
     """
     Abstraction of training and test :class:`LabelledCollection` objects.
 
@@ -545,7 +573,7 @@
     :param name: a string representing the name of the dataset
     """
 
-    def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''):
+    def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''):
         assert set(training.classes_) == set(test.classes_), 'incompatible labels in training and test collections'
         self.training = training
         self.test = test
@@ -555,7 +583,7 @@
 

 [docs]
     @classmethod
-    def SplitStratified(cls, collection: LabelledCollection, train_size=0.6):
+    def SplitStratified(cls, collection: LabelledCollection, train_size=0.6):
         """
         Generates a :class:`Dataset` from a stratified split of a :class:`LabelledCollection` instance.
         See :meth:`LabelledCollection.split_stratified`
@@ -568,7 +596,7 @@
 
 
     @property
-    def classes_(self):
+    def classes_(self):
         """
         The classes according to which the training collection is labelled
 
@@ -577,7 +605,7 @@
         return self.training.classes_
 
     @property
-    def n_classes(self):
+    def n_classes(self):
         """
         The number of classes according to which the training collection is labelled
 
@@ -586,7 +614,7 @@
         return self.training.n_classes
 
     @property
-    def binary(self):
+    def binary(self):
         """
         Returns True if the training collection is labelled according to two classes
 
@@ -597,7 +625,7 @@
 

 [docs]
     @classmethod
-    def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs):
+    def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs):
         """
         Loads a training and a test labelled set of data and convert it into a :class:`Dataset` instance.
         The function in charge of reading the instances must be specified. This function can be a custom one, or any of
@@ -619,7 +647,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         If the dataset is textual, and the vocabulary was indicated, returns the size of the vocabulary
 
@@ -628,7 +656,7 @@
         return len(self.vocabulary)
 
     @property
-    def train_test(self):
+    def train_test(self):
         """
         Alias to `self.training` and `self.test`
 
@@ -639,7 +667,7 @@
 
 

 [docs]
-    def stats(self, show=True):
+    def stats(self, show=True):
         """
         Returns (and eventually prints) a dictionary with some stats of this dataset. E.g.,:
 
@@ -666,7 +694,7 @@
 

 [docs]
     @classmethod
-    def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0):
+    def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0):
         """
         Generator of stratified folds to be used in k-fold cross validation. This function is only a wrapper around
         :meth:`LabelledCollection.kFCV` that returns :class:`Dataset` instances made of training and test folds.
@@ -683,7 +711,7 @@
 
 

 [docs]
-    def reduce(self, n_train=100, n_test=100):
+    def reduce(self, n_train=100, n_test=100, random_state=None):
         """
         Reduce the number of instances in place for quick experiments. Preserves the prevalence of each set.
 
@@ -691,10 +719,21 @@
         :param n_test: number of test documents to keep (default 100)
         :return: self
         """
-        self.training = self.training.sampling(n_train, *self.training.prevalence())
-        self.test = self.test.sampling(n_test, *self.test.prevalence())
+        self.training = self.training.sampling(
+            n_train,
+            *self.training.prevalence(),
+            random_state = random_state
+        )
+        self.test = self.test.sampling(
+            n_test,
+            *self.test.prevalence(),
+            random_state = random_state
+        )
         return self

-

+
+
+    def __repr__(self):
+        return f'training={self.training}; test={self.test}'

 
 

 
diff --git a/docs/build/html/_modules/quapy/data/datasets.html b/docs/build/html/_modules/quapy/data/datasets.html
index b910036..cbddc09 100644
--- a/docs/build/html/_modules/quapy/data/datasets.html
+++ b/docs/build/html/_modules/quapy/data/datasets.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
@@ -71,59 +74,105 @@

Source code for quapy.data.datasets

-
[docs]def warn(*args, **kwargs):
-    pass

-import warnings
-warnings.warn = warn
-import os
-import zipfile
-from os.path import join
-import pandas as pd
-from ucimlrepo import fetch_ucirepo
-from quapy.data.base import Dataset, LabelledCollection
-from quapy.data.preprocessing import text2tfidf, reduce_columns
-from quapy.data.reader import *
-from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
+import os
+from contextlib import contextmanager
+import zipfile
+from os.path import join
+import pandas as pd
+from quapy.data.base import Dataset, LabelledCollection
+from quapy.data.preprocessing import text2tfidf, reduce_columns
+from quapy.data.preprocessing import standardize as standardizer
+from quapy.data.reader import *
+from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
+from sklearn.preprocessing import StandardScaler
+
+
+def _fetch_ucirepo(*args, **kwargs):
+    try:
+        from ucimlrepo import fetch_ucirepo
+    except ImportError as exc:
+        raise ImportError(
+            "UCI dataset fetching requires the optional 'ucimlrepo' package. "
+            "Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset."
+        ) from exc
+    return fetch_ucirepo(*args, **kwargs)
 
 
 REVIEWS_SENTIMENT_DATASETS = ['hp', 'kindle', 'imdb']
-TWITTER_SENTIMENT_DATASETS_TEST = ['gasp', 'hcr', 'omd', 'sanders',
-                              'semeval13', 'semeval14', 'semeval15', 'semeval16',
-                              'sst', 'wa', 'wb']
-TWITTER_SENTIMENT_DATASETS_TRAIN = ['gasp', 'hcr', 'omd', 'sanders',
-                                 'semeval', 'semeval16',
-                                 'sst', 'wa', 'wb']
-UCI_BINARY_DATASETS = ['acute.a', 'acute.b',
-                'balance.1', 'balance.2', 'balance.3',
-                'breast-cancer',
-                'cmc.1', 'cmc.2', 'cmc.3',
-                'ctg.1', 'ctg.2', 'ctg.3',
-                       #'diabetes', # <-- I haven't found this one...
-                'german',
-                'haberman',
-                'ionosphere',
-                'iris.1', 'iris.2', 'iris.3',
-                'mammographic',
-                       'pageblocks.5',
-                       #'phoneme', # <-- I haven't found this one...
-                       'semeion',
-                       'sonar',
-                       'spambase',
-                       'spectf',
-                       'tictactoe',
-                       'transfusion',
-                       'wdbc',
-                       'wine.1', 'wine.2', 'wine.3',
-                       'wine-q-red', 'wine-q-white',
-                       'yeast']
 
-UCI_MULTICLASS_DATASETS = ['dry-bean',
-                           'wine-quality',
-                           'academic-success',
-                           'digits',
-                           'letter']
+TWITTER_SENTIMENT_DATASETS_TEST = [
+    'gasp', 'hcr', 'omd', 'sanders',
+    'semeval13', 'semeval14', 'semeval15', 'semeval16',
+    'sst', 'wa', 'wb',
+]
 
-LEQUA2022_TASKS = ['T1A', 'T1B', 'T2A', 'T2B']
+TWITTER_SENTIMENT_DATASETS_TRAIN = [
+    'gasp', 'hcr', 'omd', 'sanders',
+    'semeval', 'semeval16',
+    'sst', 'wa', 'wb',
+]
+
+UCI_BINARY_DATASETS = [
+    #'acute.a', 'acute.b',
+    'balance.1', 
+    #'balance.2', 
+    'balance.3',
+    'breast-cancer',
+    'cmc.1', 'cmc.2', 'cmc.3',
+    'ctg.1', 'ctg.2', 'ctg.3',
+    #'diabetes', # <-- I haven't found this one...
+    'german',
+    'haberman',
+    'ionosphere',
+    'iris.1', 'iris.2', 'iris.3',
+    'mammographic',
+    'pageblocks.5',
+    #'phoneme', # <-- I haven't found this one...
+    'semeion',
+    'sonar',
+    'spambase',
+    'spectf',
+    'tictactoe',
+    'transfusion',
+    'wdbc',
+    'wine.1', 'wine.2', 'wine.3',
+    'wine-q-red',
+    'wine-q-white',
+    'yeast',
+]
+
+UCI_MULTICLASS_DATASETS = [
+    'dry-bean',
+    'wine-quality',
+    'academic-success',
+    'digits',
+    'letter',
+    'abalone',
+    'obesity',
+    'nursery',
+    'yeast',
+    'hand_digits',
+    'satellite',
+    'shuttle',
+    'cmc',
+    'isolet',
+    'waveform-v1',
+    'molecular',
+    'poker_hand',
+    'connect-4',
+    'mhr',
+    'chess',
+    'page_block',
+    'phishing',
+    'image_seg',
+    'hcv',
+]
+
+LEQUA2022_VECTOR_TASKS = ['T1A', 'T1B']
+LEQUA2022_TEXT_TASKS = ['T2A', 'T2B']
+LEQUA2022_TASKS = LEQUA2022_VECTOR_TASKS + LEQUA2022_TEXT_TASKS
+
+LEQUA2024_TASKS = ['T1', 'T2', 'T3', 'T4']
 
 _TXA_SAMPLE_SIZE = 250
 _TXB_SAMPLE_SIZE = 1000
@@ -139,12 +188,22 @@
     'multiclass': _TXB_SAMPLE_SIZE
 }
 
+LEQUA2024_SAMPLE_SIZE = {
+    'T1': 250,
+    'T2': 1000,
+    'T3': 200,
+    'T4': 250,
+}
 
-
[docs]def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset:
+
+

+[docs]
+def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset:
     """
     Loads a Reviews dataset as a Dataset instance, as used in
     `Esuli, A., Moreo, A., and Sebastiani, F. "A recurrent neural network for sentiment quantification."
-    Proceedings of the 27th ACM International Conference on Information and Knowledge Management. 2018. <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_.
+    Proceedings of the 27th ACM International Conference on Information and Knowledge Management. 2018.
+    <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_.
     The list of valid dataset names can be accessed in `quapy.data.datasets.REVIEWS_SENTIMENT_DATASETS`
 
     :param dataset_name: the name of the dataset: valid ones are 'hp', 'kindle', 'imdb'
@@ -186,7 +245,10 @@
     return data

 
 
-
[docs]def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset:
+
+

+[docs]
+def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset:
     """
     Loads a Twitter dataset as a :class:`quapy.data.base.Dataset` instance, as used in:
     `Gao, W., Sebastiani, F.: From classification to quantification in tweet sentiment analysis.
@@ -260,7 +322,10 @@
     return data

 
 
-
[docs]def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, verbose=False) -> Dataset:
+
+

+[docs]
+def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, standardize=True, verbose=False) -> Dataset:
     """
     Loads a UCI dataset as an instance of :class:`quapy.data.base.Dataset`, as used in
     `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).
@@ -278,14 +343,23 @@
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
     :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,
+        standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the
+        training portion of the data.
     :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets
     :return: a :class:`quapy.data.base.Dataset` instance
     """
     data = fetch_UCIBinaryLabelledCollection(dataset_name, data_home, verbose)
-    return Dataset(*data.split_stratified(1 - test_split, random_state=0))

+    dataset = Dataset(*data.split_stratified(1 - test_split, random_state=0), name=dataset_name)
+    if standardize:
+        dataset = standardizer(dataset)
+    return dataset

 
 
-
[docs]def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, verbose=False) -> LabelledCollection:
+
+

+[docs]
+def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize=True, verbose=False) -> LabelledCollection:
     """
     Loads a UCI collection as an instance of :class:`quapy.data.base.LabelledCollection`, as used in
     `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).
@@ -301,7 +375,7 @@
 
     >>> import quapy as qp
     >>> collection = qp.datasets.fetch_UCIBinaryLabelledCollection("yeast")
-    >>> for data in qp.train.Dataset.kFCV(collection, nfolds=5, nrepeats=2):
+    >>> for data in qp.datasets.Dataset.kFCV(collection, nfolds=5, nrepeats=2):
     >>>     ...
 
     The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_DATASETS`
@@ -309,327 +383,312 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). 
     :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets
     :return: a :class:`quapy.data.base.LabelledCollection` instance
     """
-
-    assert dataset_name in UCI_BINARY_DATASETS, \
-        f'Name {dataset_name} does not match any known dataset from the UCI Machine Learning datasets repository. ' \
-        f'Valid ones are {UCI_BINARY_DATASETS}'
+    assert dataset_name in UCI_BINARY_DATASETS, (
+        f"Name {dataset_name} does not match any known dataset from the UCI Machine Learning datasets repository. "
+        f"Valid ones are {UCI_BINARY_DATASETS}"
+    )
     if data_home is None:
         data_home = get_quapy_home()
 
-    dataset_fullname = {
-        'acute.a': 'Acute Inflammations (urinary bladder)',
-        'acute.b': 'Acute Inflammations (renal pelvis)',
-        'balance.1': 'Balance Scale Weight & Distance Database (left)',
-        'balance.2': 'Balance Scale Weight & Distance Database (balanced)',
-        'balance.3': 'Balance Scale Weight & Distance Database (right)',
-        'breast-cancer':  'Breast Cancer Wisconsin (Original)',
-        'cmc.1': 'Contraceptive Method Choice (no use)',
-        'cmc.2': 'Contraceptive Method Choice (long term)',
-        'cmc.3': 'Contraceptive Method Choice (short term)',
-        'ctg.1': 'Cardiotocography Data Set (normal)',
-        'ctg.2': 'Cardiotocography Data Set (suspect)',
-        'ctg.3': 'Cardiotocography Data Set (pathologic)',
-        'german': 'Statlog German Credit Data',
-        'haberman': "Haberman's Survival Data",
-        'ionosphere': 'Johns Hopkins University Ionosphere DB',
-        'iris.1': 'Iris Plants Database(x)',
-        'iris.2': 'Iris Plants Database(versicolour)',
-        'iris.3': 'Iris Plants Database(virginica)',
-        'mammographic': 'Mammographic Mass',
-        'pageblocks.5': 'Page Blocks Classification (5)',
-        'semeion': 'Semeion Handwritten Digit (8)',
-        'sonar': 'Sonar, Mines vs. Rocks',
-        'spambase': 'Spambase Data Set',
-        'spectf': 'SPECTF Heart Data',
-        'tictactoe': 'Tic-Tac-Toe Endgame Database',
-        'transfusion': 'Blood Transfusion Service Center Data Set',
-        'wdbc': 'Wisconsin Diagnostic Breast Cancer',
-        'wine.1': 'Wine Recognition Data (1)',
-        'wine.2': 'Wine Recognition Data (2)',
-        'wine.3': 'Wine Recognition Data (3)',
-        'wine-q-red': 'Wine Quality Red (6-10)',
-        'wine-q-white': 'Wine Quality White (6-10)',
-        'yeast': 'Yeast',
+    # mapping bewteen dataset names and UCI api ids
+    identifiers = {
+        "acute.a": 184,
+        "acute.b": 184,
+        "balance.1": 12,
+        "balance.2": 12,
+        "balance.3": 12,
+        "breast-cancer": 15,
+        "cmc.1": 30,
+        "cmc.2": 30,
+        "cmc.3": 30,
+        # "ctg.1": ,  # not python importable
+        # "ctg.2": ,  # not python importable
+        # "ctg.3": ,  # not python importable
+        # "german": ,  # not python importable
+        "haberman": 43,
+        "ionosphere": 52,
+        "iris.1": 53,
+        "iris.2": 53,
+        "iris.3": 53,
+        "mammographic": 161,
+        "pageblocks.5": 78,
+        # "semeion": ,  # not python importable
+        "sonar": 151,
+        "spambase": 94,
+        "spectf": 96,
+        "tictactoe": 101,
+        "transfusion": 176,
+        "wdbc": 17,
+        "wine.1": 109,
+        "wine.2": 109,
+        "wine.3": 109,
+        "wine-q-red": 186,
+        "wine-q-white": 186,
+        "yeast": 110,
     }
 
-    # the identifier is an alias for the dataset group, it's part of the url data-folder, and is the name we use
-    # to download the raw dataset
-    identifier_map = {
-        'acute.a': 'acute',
-        'acute.b': 'acute',
-        'balance.1': 'balance-scale',
-        'balance.2': 'balance-scale',
-        'balance.3': 'balance-scale',
-        'breast-cancer': 'breast-cancer-wisconsin',
-        'cmc.1': 'cmc',
-        'cmc.2': 'cmc',
-        'cmc.3': 'cmc',
-        'ctg.1': '00193',
-        'ctg.2': '00193',
-        'ctg.3': '00193',
-        'german': 'statlog/german',
-        'haberman': 'haberman',
-        'ionosphere': 'ionosphere',
-        'iris.1': 'iris',
-        'iris.2': 'iris',
-        'iris.3': 'iris',
-        'mammographic': 'mammographic-masses',
-        'pageblocks.5': 'page-blocks',
-        'semeion': 'semeion',
-        'sonar': 'undocumented/connectionist-bench/sonar',
-        'spambase': 'spambase',
-        'spectf': 'spect',
-        'tictactoe': 'tic-tac-toe',
-        'transfusion': 'blood-transfusion',
-        'wdbc': 'breast-cancer-wisconsin',
-        'wine-q-red': 'wine-quality',
-        'wine-q-white': 'wine-quality',
-        'wine.1': 'wine',
-        'wine.2': 'wine',
-        'wine.3': 'wine',
-        'yeast': 'yeast',
+    # mapping between dataset names and dataset groups
+    groups = {
+        "acute.a": "acute",
+        "acute.b": "acute",
+        "balance.1": "balance",
+        "balance.2": "balance",
+        "balance.3": "balance",
+        "breast-cancer": "breast-cancer",
+        "cmc.1": "cmc",
+        "cmc.2": "cmc",
+        "cmc.3": "cmc",
+        "ctg.1": "ctg",
+        "ctg.2": "ctg",
+        "ctg.3": "ctg",
+        "german": "german",
+        "haberman": "haberman",
+        "ionosphere": "ionosphere",
+        "iris.1": "iris",
+        "iris.2": "iris",
+        "iris.3": "iris",
+        "mammographic": "mammographic",
+        "pageblocks.5": "pageblocks",
+        "semeion": "semeion",
+        "sonar": "sonar",
+        "spambase": "spambase",
+        "spectf": "spectf",
+        "tictactoe": "tictactoe",
+        "transfusion": "transfusion",
+        "wdbc": "wdbc",
+        "wine-q-red": "wine-quality",
+        "wine-q-white": "wine-quality",
+        "wine.1": "wine",
+        "wine.2": "wine",
+        "wine.3": "wine",
+        "yeast": "yeast",
     }
 
-    # the filename is the name of the file within the data_folder indexed by the identifier
-    file_name = {
-        'acute': 'diagnosis.data',
-        '00193': 'CTG.xls',
-        'statlog/german': 'german.data-numeric',
-        'mammographic-masses': 'mammographic_masses.data',
-        'page-blocks': 'page-blocks.data.Z',
-        'undocumented/connectionist-bench/sonar': 'sonar.all-data',
-        'spect': ['SPECTF.train', 'SPECTF.test'],
-        'blood-transfusion': 'transfusion.data',
-        'wine-quality': ['winequality-red.csv', 'winequality-white.csv'],
-        'breast-cancer-wisconsin': 'breast-cancer-wisconsin.data' if dataset_name=='breast-cancer' else 'wdbc.data'
+    # mapping between dataset short names and full names
+    full_names = {
+        "acute.a": "Acute Inflammations (urinary bladder)",
+        "acute.b": "Acute Inflammations (renal pelvis)",
+        "balance.1": "Balance Scale Weight & Distance Database (left)",
+        "balance.2": "Balance Scale Weight & Distance Database (balanced)",
+        "balance.3": "Balance Scale Weight & Distance Database (right)",
+        "breast-cancer": "Breast Cancer Wisconsin (Original)",
+        "cmc.1": "Contraceptive Method Choice (no use)",
+        "cmc.2": "Contraceptive Method Choice (long term)",
+        "cmc.3": "Contraceptive Method Choice (short term)",
+        "ctg.1": "Cardiotocography Data Set (normal)",
+        "ctg.2": "Cardiotocography Data Set (suspect)",
+        "ctg.3": "Cardiotocography Data Set (pathologic)",
+        "german": "Statlog German Credit Data",
+        "haberman": "Haberman's Survival Data",
+        "ionosphere": "Johns Hopkins University Ionosphere DB",
+        "iris.1": "Iris Plants Database(x)",
+        "iris.2": "Iris Plants Database(versicolour)",
+        "iris.3": "Iris Plants Database(virginica)",
+        "mammographic": "Mammographic Mass",
+        "pageblocks.5": "Page Blocks Classification (5)",
+        "semeion": "Semeion Handwritten Digit (8)",
+        "sonar": "Sonar, Mines vs. Rocks",
+        "spambase": "Spambase Data Set",
+        "spectf": "SPECTF Heart Data",
+        "tictactoe": "Tic-Tac-Toe Endgame Database",
+        "transfusion": "Blood Transfusion Service Center Data Set",
+        "wdbc": "Wisconsin Diagnostic Breast Cancer",
+        "wine.1": "Wine Recognition Data (1)",
+        "wine.2": "Wine Recognition Data (2)",
+        "wine.3": "Wine Recognition Data (3)",
+        "wine-q-red": "Wine Quality Red (6-10)",
+        "wine-q-white": "Wine Quality White (6-10)",
+        "yeast": "Yeast",
     }
 
-    # the filename containing the dataset description (if any)
-    desc_name = {
-        'acute': 'diagnosis.names',
-        '00193': None,
-        'statlog/german': 'german.doc',
-        'mammographic-masses': 'mammographic_masses.names',
-        'undocumented/connectionist-bench/sonar': 'sonar.names',
-        'spect': 'SPECTF.names',
-        'blood-transfusion': 'transfusion.names',
-        'wine-quality': 'winequality.names',
-        'breast-cancer-wisconsin': 'breast-cancer-wisconsin.names' if dataset_name == 'breast-cancer' else 'wdbc.names'
+    # mapping between dataset names and values of positive class
+    pos_class = {
+        "acute.a": "yes",
+        "acute.b": "yes",
+        "balance.1": "L",
+        "balance.2": "B",
+        "balance.3": "R",
+        "breast-cancer": 2,
+        "cmc.1": 1,
+        "cmc.2": 2,
+        "cmc.3": 3,
+        "ctg.1": 1,  # 1==Normal
+        "ctg.2": 2,  # 2==Suspect
+        "ctg.3": 3,  # 3==Pathologic
+        "german": 1,
+        "haberman": 2,
+        "ionosphere": "b",
+        "iris.1": "Iris-setosa",  # 1==Setosa
+        "iris.2": "Iris-versicolor",  # 2==Versicolor
+        "iris.3": "Iris-virginica",  # 3==Virginica
+        "mammographic": 1,
+        "pageblocks.5": 5,  # 5==block "graphic"
+        "semeion": 1,
+        "sonar": "R",
+        "spambase": 1,
+        "spectf": 0,
+        "tictactoe": "negative",
+        "transfusion": 1,
+        "wdbc": "M",
+        "wine.1": 1,
+        "wine.2": 2,
+        "wine.3": 3,
+        "wine-q-red": 1,
+        "wine-q-white": 1,
+        "yeast": "NUC",
     }
 
-    identifier = identifier_map[dataset_name]
-    filename = file_name.get(identifier, f'{identifier}.data')
-    descfile = desc_name.get(identifier, f'{identifier}.names')
-    fullname = dataset_fullname[dataset_name]
-
-    URL = f'http://archive.ics.uci.edu/ml/machine-learning-databases/{identifier}'
-    data_dir = join(data_home, 'uci_datasets', identifier)
-    if isinstance(filename, str):  # filename could be a list of files, in which case it will be processed later
-        data_path = join(data_dir, filename)
-        download_file_if_not_exists(f'{URL}/{filename}', data_path)
-
-    if descfile:
-        try:
-            download_file_if_not_exists(f'{URL}/{descfile}', f'{data_dir}/{descfile}')
-            if verbose:
-                print(open(f'{data_dir}/{descfile}', 'rt').read())
-        except Exception:
-            print('could not read the description file')
-    elif verbose:
-        print('no file description available')
+    identifier = identifiers.get(dataset_name, None)
+    dataset_group = groups[dataset_name]
+    fullname = full_names[dataset_name]
 
     if verbose:
-        print(f'Loading {dataset_name} ({fullname})')
-    if identifier == 'acute':
-        df = pd.read_csv(data_path, header=None, encoding='utf-16', sep='\t')
+        print(f"Loading UCI Binary {dataset_name} ({fullname})")
 
-        df[0] = df[0].apply(lambda x: float(x.replace(',', '.'))).astype(float, copy=False)
-        [_df_replace(df, col) for col in range(1, 6)]
-        X = df.loc[:, 0:5].values
-        if dataset_name == 'acute.a':
-            y = binarize(df[6], pos_class='yes')
-        elif dataset_name == 'acute.b':
-            y = binarize(df[7], pos_class='yes')
+    file = join(data_home, "uci_datasets", dataset_group + ".pkl")
 
-    if identifier == 'balance-scale':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        if dataset_name == 'balance.1':
-            y = binarize(df[0], pos_class='L')
-        elif dataset_name == 'balance.2':
-            y = binarize(df[0], pos_class='B')
-        elif dataset_name == 'balance.3':
-            y = binarize(df[0], pos_class='R')
-        X = df.loc[:, 1:].astype(float).values
+    @contextmanager
+    def download_tmp_file(url_group: str, filename: str):
+        """
+        Download a data file for a group of datasets temporarely.
+        When used as a context, the file is removed once the context exits.
 
-    if identifier == 'breast-cancer-wisconsin' and dataset_name=='breast-cancer':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        Xy = df.loc[:, 1:10]
-        Xy[Xy=='?']=np.nan
-        Xy = Xy.dropna(axis=0)
-        X = Xy.loc[:, 1:9]
-        X = X.astype(float).values
-        y = binarize(Xy[10], pos_class=2)
-
-    if identifier == 'breast-cancer-wisconsin' and dataset_name=='wdbc':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.loc[:, 2:32].astype(float).values
-        y = df[1].values
-        y = binarize(y, pos_class='M')
-
-    if identifier == 'cmc':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.loc[:, 0:8].astype(float).values
-        y = df[9].astype(int).values
-        if dataset_name == 'cmc.1':
-            y = binarize(y, pos_class=1)
-        elif dataset_name == 'cmc.2':
-            y = binarize(y, pos_class=2)
-        elif dataset_name == 'cmc.3':
-            y = binarize(y, pos_class=3)
-
-    if identifier == '00193':
-        df = pd.read_excel(data_path, sheet_name='Data', skipfooter=3)
-        df = df[list(range(1,24))] # select columns numbered (number 23 is the target label)
-        # replaces the header with the first row
-        new_header = df.iloc[0]  # grab the first row for the header
-        df = df[1:]  # take the data less the header row
-        df.columns = new_header  # set the header row as the df header
-        X = df.iloc[:, 0:22].astype(float).values
-        y = df['NSP'].astype(int).values
-        if dataset_name == 'ctg.1':
-            y = binarize(y, pos_class=1)  # 1==Normal
-        elif dataset_name == 'ctg.2':
-            y = binarize(y, pos_class=2)  # 2==Suspect
-        elif dataset_name == 'ctg.3':
-            y = binarize(y, pos_class=3)  # 3==Pathologic
-
-    if identifier == 'statlog/german':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True)
-        X = df.iloc[:, 0:24].astype(float).values
-        y = df[24].astype(int).values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'haberman':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:3].astype(float).values
-        y = df[3].astype(int).values
-        y = binarize(y, pos_class=2)
-
-    if identifier == 'ionosphere':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:34].astype(float).values
-        y = df[34].values
-        y = binarize(y, pos_class='b')
-
-    if identifier == 'iris':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:4].astype(float).values
-        y = df[4].values
-        if dataset_name == 'iris.1':
-            y = binarize(y, pos_class='Iris-setosa')  # 1==Setosa
-        elif dataset_name == 'iris.2':
-            y = binarize(y, pos_class='Iris-versicolor')  # 2==Versicolor
-        elif dataset_name == 'iris.3':
-            y = binarize(y, pos_class='Iris-virginica')  # 3==Virginica
-
-    if identifier == 'mammographic-masses':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        df[df == '?'] = np.nan
-        Xy = df.dropna(axis=0)
-        X = Xy.iloc[:, 0:5]
-        X = X.astype(float).values
-        y = binarize(Xy.iloc[:,5], pos_class=1)
-
-    if identifier == 'page-blocks':
-        data_path_ = data_path.replace('.Z', '')
-        if not os.path.exists(data_path_):
-            raise FileNotFoundError(f'Warning: file {data_path_} does not exist. If this is the first time you '
-                                    f'attempt to load this dataset, then you have to manually unzip the {data_path} '
-                                    f'and name the extracted file {data_path_} (unfortunately, neither zipfile, nor '
-                                    f'gzip can handle unix compressed files automatically -- there is a repo in GitHub '
-                                    f'https://github.com/umeat/unlzw where the problem seems to be solved anyway).')
-        df = pd.read_csv(data_path_, header=None, delim_whitespace=True)
-        X = df.iloc[:, 0:10].astype(float).values
-        y = df[10].values
-        y = binarize(y, pos_class=5)  # 5==block "graphic"
-
-    if identifier == 'semeion':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True )
-        X = df.iloc[:, 0:256].astype(float).values
-        y = df[263].values  # 263 stands for digit 8 (labels are one-hot vectors from col 256-266)
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'undocumented/connectionist-bench/sonar':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:60].astype(float).values
-        y = df[60].values
-        y = binarize(y, pos_class='R')
-
-    if identifier == 'spambase':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:57].astype(float).values
-        y = df[57].values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'spect':
-        dfs = []
-        for file in filename:
-            data_path = join(data_dir, file)
-            download_file_if_not_exists(f'{URL}/{file}', data_path)
-            dfs.append(pd.read_csv(data_path, header=None, sep=','))
-        df = pd.concat(dfs)
-        X = df.iloc[:, 1:45].astype(float).values
-        y = df[0].values
-        y = binarize(y, pos_class=0)
-
-    if identifier == 'tic-tac-toe':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:9].replace('o',0).replace('b',1).replace('x',2).values
-        y = df[9].values
-        y = binarize(y, pos_class='negative')
-
-    if identifier == 'blood-transfusion':
-        df = pd.read_csv(data_path, sep=',')
-        X = df.iloc[:, 0:4].astype(float).values
-        y = df.iloc[:, 4].values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'wine':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 1:14].astype(float).values
-        y = df[0].values
-        if dataset_name == 'wine.1':
-            y = binarize(y, pos_class=1)
-        elif dataset_name == 'wine.2':
-            y = binarize(y, pos_class=2)
-        elif dataset_name == 'wine.3':
-            y = binarize(y, pos_class=3)
-
-    if identifier == 'wine-quality':
-        filename = filename[0] if dataset_name=='wine-q-red' else filename[1]
+        :param url_group: identifier of the dataset group in the URL
+        :param filename: name of the file to be downloaded
+        """
+        data_dir = join(data_home, "uci_datasets", "tmp")
+        os.makedirs(data_dir, exist_ok=True)
         data_path = join(data_dir, filename)
-        download_file_if_not_exists(f'{URL}/{filename}', data_path)
-        df = pd.read_csv(data_path, sep=';')
-        X = df.iloc[:, 0:11].astype(float).values
-        y = df.iloc[:, 11].values > 5
+        url = f"http://archive.ics.uci.edu/ml/machine-learning-databases/{url_group}/{filename}"
+        download_file_if_not_exists(url, data_path)
+        try:
+            yield data_path
+        finally:
+            os.remove(data_path)
 
-    if identifier == 'yeast':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True)
-        X = df.iloc[:, 1:9].astype(float).values
-        y = df.iloc[:, 9].values
-        y = binarize(y, pos_class='NUC')
+    def download(id: int | None, group: str) -> dict:
+        """
+        Download the data to be pickled for a dataset group. Use the `fetch_ucirepo` api when possible.
 
-    data = LabelledCollection(X, y)
+        :param id: numeric identifier for the group; can be None
+        :param group: group name
+        :return: a dictionary with X and y as keys and, optionally, extra data.
+        """
+
+        # use the fetch_ucirepo api, when possible, to download data
+        # fall back to direct download when needed
+        if group == "german":
+            with download_tmp_file("statlog/german", "german.data-numeric") as tmp:
+                df = pd.read_csv(tmp, header=None, delim_whitespace=True)
+            X, y = df.iloc[:, 0:24].astype(float).values, df[24].astype(int).values
+        elif group == "ctg":
+            with download_tmp_file("00193", "CTG.xls") as tmp:
+                df = pd.read_excel(tmp, sheet_name="Data", skipfooter=3)
+            df = df[list(range(1, 24))]  # select columns numbered (number 23 is the target label)
+            # replaces the header with the first row
+            new_header = df.iloc[0]  # grab the first row for the header
+            df = df[1:]  # take the data less the header row
+            df.columns = new_header  # set the header row as the df header
+            X = df.iloc[:, 0:21].astype(float).values  # column 21 is skipped, it is a class column
+            y = df["NSP"].astype(int).values
+        elif group == "semeion":
+            with download_tmp_file("semeion", "semeion.data") as tmp:
+                df = pd.read_csv(tmp, header=None, sep='\\s+')
+            X = df.iloc[:, 0:256].astype(float).values
+            y = df[263].values  # 263 stands for digit 8 (labels are one-hot vectors from col 256-266)
+        else:
+            df = _fetch_ucirepo(id=id)
+            X, y = df.data.features.to_numpy(), df.data.targets.to_numpy().squeeze()
+
+        # transform data when needed before returning (returned data will be pickled)
+        if group == "acute":
+            _array_replace(X)
+            data = {"X": X, "y": y}
+        elif group == "balance":
+            # features' order is reversed to match data retrieved via direct download
+            X = X[:, np.arange(X.shape[1])[::-1]]
+            data = {"X": X, "y": y}
+        elif group == "breast-cancer":
+            # remove rows with nan values
+            Xy = np.hstack([X, y[:, np.newaxis]])
+            nan_rows = np.isnan(Xy).sum(axis=-1) > 0
+            Xy = Xy[~nan_rows]
+            data = {"X": Xy[:, :-1], "y": Xy[:, -1]}
+        elif group == "mammographic":
+            # remove rows with nan values
+            Xy = np.hstack([X, y[:, np.newaxis]])
+            nan_rows = np.isnan(Xy).sum(axis=-1) > 0
+            Xy = Xy[~nan_rows]
+            data = {"X": Xy[:, :-1], "y": Xy[:, -1]}
+        elif group == "tictactoe":
+            _array_replace(X, repl={"o": 0, "b": 1, "x": 2})
+            data = {"X": X, "y": y}
+        elif group == "wine-quality":
+            # add color data to split the final datasets
+            color = df.data.original["color"].to_numpy()
+            data = {"X": X, "y": y, "color": color}
+        else:
+            data = {"X": X, "y": y}
+
+        return data
+
+    def binarize_data(name, data: dict) -> LabelledCollection:
+        """
+        Filter and transform data to extract a binary dataset.
+
+        :param name: name of the dataset
+        :param data: dictionary containing X and y fields, plus additional data when needed
+        :return: a :class:`quapy.data.base.LabelledCollection` with the extracted dataset
+        """
+        if name == "acute.a":
+            X, y = data["X"], data["y"][:, 0]
+        elif name == "acute.b":
+            X, y = data["X"], data["y"][:, 1]
+        elif name == "wine-q-red":
+            X, y, color = data["X"], data["y"], data["color"]
+            red_idx = color == "red"
+            X, y = X[red_idx, :], y[red_idx]
+            y = (y > 5).astype(int)
+        elif name == "wine-q-white":
+            X, y, color = data["X"], data["y"], data["color"]
+            white_idx = color == "white"
+            X, y = X[white_idx, :], y[white_idx]
+            y = (y > 5).astype(int)
+        else:
+            X, y = data["X"], data["y"]
+
+        y = binarize(y, pos_class=pos_class[name])
+
+        return LabelledCollection(X, y)
+
+    data = pickled_resource(file, download, identifier, dataset_group)
+    data = binarize_data(dataset_name, data)
+
+    if standardize:
+        stds = StandardScaler()
+        data.instances = stds.fit_transform(data.instances)
+        
     if verbose:
         data.stats()
+
     return data

 
 
-
[docs]def fetch_UCIMulticlassDataset(dataset_name, data_home=None, test_split=0.3, verbose=False) -> Dataset:
+
+

+[docs]
+def fetch_UCIMulticlassDataset(
+        dataset_name,
+        data_home=None,
+        min_test_split=0.3,
+        max_train_instances=25000,
+        min_class_support=100,
+        standardize=True,
+        verbose=False) -> Dataset:
     """
     Loads a UCI multiclass dataset as an instance of :class:`quapy.data.base.Dataset`. 
 
@@ -651,15 +710,41 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param min_test_split: minimum proportion of instances to be included in the test set. This value is interpreted
+        as a minimum proportion, meaning that the real proportion could be higher in case the training proportion
+        (1-`min_test_split`% of the instances) surpasses `max_train_instances`. In such case, only `max_train_instances`
+        are taken for training, and the rest (irrespective of `min_test_split`) is taken for test.
+    :param max_train_instances: maximum number of instances to keep for training (defaults to 25000);
+        set to -1 or None to avoid this check
+    :param min_class_support: integer or float, the minimum number or proportion of istances per class.
+        Classes with fewer instances are discarded (deafult is 100).
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,
+        standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the
+        training portion of the data.
     :param verbose: set to True (default is False) to get information (stats) about the dataset
     :return: a :class:`quapy.data.base.Dataset` instance
     """
-    data = fetch_UCIMulticlassLabelledCollection(dataset_name, data_home, verbose)
-    return Dataset(*data.split_stratified(1 - test_split, random_state=0))

+
+    data = fetch_UCIMulticlassLabelledCollection(dataset_name, data_home, min_class_support, verbose=verbose)
+    n = len(data)
+    train_prop = (1.-min_test_split)
+    if (max_train_instances is not None) and (max_train_instances > 0):
+        n_train = int(n*train_prop)
+        if n_train > max_train_instances:
+            train_prop = (max_train_instances / n)
+
+    data = Dataset(*data.split_stratified(train_prop, random_state=0), name=dataset_name)
+    
+    if standardize:
+        data = standardizer(data)
+    
+    return data

 
 
-
[docs]def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, verbose=False) -> LabelledCollection:
+
+

+[docs]
+def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_class_support=100, standardize=True, verbose=False) -> LabelledCollection:
     """
     Loads a UCI multiclass collection as an instance of :class:`quapy.data.base.LabelledCollection`.
 
@@ -681,7 +766,9 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where the dataset will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param min_class_support: minimum number of istances per class. Classes with fewer instances
+        are discarded (deafult is 100)
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). 
     :param verbose: set to True (default is False) to get information (stats) about the dataset
     :return: a :class:`quapy.data.base.LabelledCollection` instance
     """
@@ -689,24 +776,67 @@
         f'Name {dataset_name} does not match any known dataset from the ' \
         f'UCI Machine Learning datasets repository (multiclass). ' \
         f'Valid ones are {UCI_MULTICLASS_DATASETS}'
+
+    assert (min_class_support is None or
+            ((isinstance(min_class_support, int) and min_class_support >= 0) or
+             (isinstance(min_class_support, float) and 0. <= min_class_support < 1.))), \
+        f'invalid value for {min_class_support=}; expected non negative integer or float in [0,1)'
     
     if data_home is None:
         data_home = get_quapy_home()
     
     identifiers = {
-        "dry-bean": 602,
-        "wine-quality": 186,
-        "academic-success": 697,
-        "digits": 80,
-        "letter": 59
+        'dry-bean': 602,
+        'wine-quality': 186,
+        'academic-success': 697,
+        'digits': 80,
+        'letter': 59,
+        'abalone': 1,
+        'obesity': 544,
+        'nursery': 76,
+        'yeast': 110,
+        'hand_digits': 81,
+        'satellite': 146,
+        'shuttle': 148,
+        'cmc': 30,
+        'isolet': 54,
+        'waveform-v1': 107,
+        'molecular': 69,
+        'poker_hand': 158,
+        'connect-4': 26,
+        'mhr': 863,
+        'chess': 23,
+        'page_block': 78,
+        'phishing': 379,
+        'image_seg': 147,
+        'hcv': 503,
     }
     
     full_names = {
-        "dry-bean": "Dry Bean Dataset",
-        "wine-quality": "Wine Quality",
-        "academic-success": "Predict students' dropout and academic success",
-        "digits": "Optical Recognition of Handwritten Digits",
-        "letter": "Letter Recognition"
+        'dry-bean': 'Dry Bean Dataset',
+        'wine-quality': 'Wine Quality',
+        'academic-success': 'Predict students\' dropout and academic success',
+        'digits': 'Optical Recognition of Handwritten Digits',
+        'letter': 'Letter Recognition',
+        'abalone': 'Abalone',
+        'obesity': 'Estimation of Obesity Levels Based On Eating Habits and Physical Condition',
+        'nursery': 'Nursery',
+        'yeast': 'Yeast',
+        'hand_digits': 'Pen-Based Recognition of Handwritten Digits',
+        'satellite': 'Statlog Landsat Satellite',
+        'shuttle': 'Statlog Shuttle',
+        'cmc': 'Contraceptive Method Choice',
+        'isolet': 'ISOLET',
+        'waveform-v1': 'Waveform Database Generator (Version 1)',
+        'molecular': 'Molecular Biology (Splice-junction Gene Sequences)',
+        'poker_hand': 'Poker Hand',
+        'connect-4': 'Connect-4',
+        'mhr': 'Maternal Health Risk',
+        'chess': 'Chess (King-Rook vs. King)',
+        'page_block': 'Page Blocks Classification',
+        'phishing': 'Website Phishing',
+        'image_seg': 'Statlog (Image Segmentation)',
+        'hcv': 'Hepatitis C Virus (HCV) for Egyptian patients',
     }
     
     identifier = identifiers[dataset_name]
@@ -717,14 +847,59 @@
 
     file = join(data_home, 'uci_multiclass', dataset_name+'.pkl')
     
-    def download(id):
-        data = fetch_ucirepo(id=id)
-        X, y = data['data']['features'].to_numpy(), data['data']['targets'].to_numpy().squeeze()
+    def dummify_categorical_features(df_features, dataset_id):
+        categorical_features = {
+            158: ["S1", "C1", "S2", "C2", "S3", "C3", "S4", "C4", "S5", "C5"],  # poker_hand
+        }
+
+        categorical = categorical_features.get(dataset_id, [])
+
+        X = df_features.copy()
+        if categorical:
+            X[categorical] = X[categorical].astype("category")
+            X = pd.get_dummies(X, columns=categorical, drop_first=True)
+
+        return X
+
+    def download(id, name):
+        df = _fetch_ucirepo(id=id)
+
+        X_df = dummify_categorical_features(df.data.features, id)
+        X = X_df.to_numpy(dtype=np.float64)
+        y = df.data.targets.to_numpy().squeeze()
+
+        assert y.ndim == 1, f'error: the dataset {id=} {name=} has more than one target variable'
+
         classes = np.sort(np.unique(y))
         y = np.searchsorted(classes, y)
         return LabelledCollection(X, y)
 
-    data = pickled_resource(file, download, identifier)
+    def filter_classes(data: LabelledCollection, min_class_support):
+        if min_class_support is None or min_class_support == 0.:
+            return data
+        if isinstance(min_class_support, float):
+            min_class_support = int(len(data) * min_class_support)
+        classes = data.classes_
+        # restrict classes to only those with at least min_class_support instances
+        classes = classes[data.counts() >= min_class_support]
+        # filter X and y keeping only datapoints belonging to valid classes
+        filter_idx = np.isin(data.y, classes)
+        X, y = data.X[filter_idx], data.y[filter_idx]
+        # map classes to range(len(classes))
+        y = np.searchsorted(classes, y)
+        return LabelledCollection(X, y)
+
+    data = pickled_resource(file, download, identifier, dataset_name)
+    data = filter_classes(data, min_class_support)
+    if data.n_classes <= 2:
+        raise ValueError(
+            f'After filtering out classes with less than {min_class_support=} instances, the dataset {dataset_name} '
+            f'is no longer multiclass. Try a reducing this value.'
+        )
+
+    if standardize:
+        stds = StandardScaler()
+        data.instances = stds.fit_transform(data.instances)
 
     if verbose:
         data.stats()
@@ -732,13 +907,21 @@
     return data

 
 
-def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float):
+
+def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float):
     df[col] = df[col].apply(lambda x:repl[x]).astype(astype, copy=False)
 
 
-
[docs]def fetch_lequa2022(task, data_home=None):
+def _array_replace(arr, repl={"yes": 1, "no": 0}):
+    for k, v in repl.items():
+        arr[arr == k] = v
+
+
+

+[docs]
+def fetch_lequa2022(task, data_home=None):
     """
-    Loads the official datasets provided for the `LeQua <https://lequa2022.github.io/index>`_ competition.
+    Loads the official datasets provided for the `LeQua 2022 <https://lequa2022.github.io/index>`_ competition.
     In brief, there are 4 tasks (T1A, T1B, T2A, T2B) having to do with text quantification
     problems. Tasks T1A and T1B provide documents in vector form, while T2A and T2B provide raw documents instead.
     Tasks T1A and T2A are binary sentiment quantification problems, while T2A and T2B are multiclass quantification
@@ -750,20 +933,19 @@
 
     The datasets are downloaded only once, and stored for fast reuse.
 
-    See `lequa2022_experiments.py` provided in the example folder, that can serve as a guide on how to use these
+    See `4.lequa2022_experiments.py` provided in the example folder, that can serve as a guide on how to use these
     datasets.
 
-
     :param task: a string representing the task name; valid ones are T1A, T1B, T2A, and T2B
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
     :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of
         :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of
-        :class:`quapy.data._lequa2022.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
+        :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
         that return a series of samples stored in a directory which are labelled by prevalence.
     """
 
-    from quapy.data._lequa2022 import load_raw_documents, load_vector_documents, SamplesFromDir
+    from quapy.data._lequa import load_raw_documents, load_vector_documents_2022, SamplesFromDir
 
     assert task in LEQUA2022_TASKS, \
         f'Unknown task {task}. Valid ones are {LEQUA2022_TASKS}'
@@ -777,11 +959,13 @@
     lequa_dir = join(data_home, 'lequa2022')
     os.makedirs(lequa_dir, exist_ok=True)
 
-    def download_unzip_and_remove(unzipped_path, url):
+    def download_unzip_and_remove(unzipped_path, url):
         tmp_path = join(lequa_dir, task + '_tmp.zip')
         download_file_if_not_exists(url, tmp_path)
         with zipfile.ZipFile(tmp_path) as file:
+            print(f'Unzipping {tmp_path}...', end='')
             file.extractall(unzipped_path)
+            print(f'[done]')
         os.remove(tmp_path)
 
     if not os.path.exists(join(lequa_dir, task)):
@@ -790,7 +974,7 @@
         download_unzip_and_remove(lequa_dir, URL_TEST_PREV)
 
     if task in ['T1A', 'T1B']:
-        load_fn = load_vector_documents
+        load_fn = load_vector_documents_2022
     elif task in ['T2A', 'T2B']:
         load_fn = load_raw_documents
 
@@ -808,14 +992,107 @@
     return train, val_gen, test_gen

 
 
-
[docs]def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None):
+
+

+[docs]
+def fetch_lequa2024(task, data_home=None, merge_T3=False):
+    """
+    Loads the official datasets provided for the `LeQua 2024 <https://lequa2024.github.io/index>`_ competition.
+    LeQua 2024 defines four tasks (T1, T2, T3, T4) related to the problem of quantification;
+    all tasks are affected by some type of dataset shift. Tasks T1 and T2 are akin to tasks T1A and T1B of LeQua 2022,
+    while T3 and T4 are new tasks introduced in LeQua 2024.
+
+    - Task T1 evaluates binary quantifiers under prior probability shift (akin to T1A of LeQua 2022).
+    - Task T2 evaluates single-label multi-class quantifiers (for n > 2 classes) under prior probability shift (akin to T1B of LeQua 2022).
+    - Task T3 evaluates ordinal quantifiers, where the classes are totally ordered.
+    - Task T4 also evaluates binary quantifiers, but under some mix of covariate shift and prior probability shift.
+
+    For a broader discussion, we refer to the `online official documentation <https://lequa2024.github.io/tasks/>`_
+
+    The datasets are downloaded only once, and stored locally for future reuse.
+
+    See `4b.lequa2024_experiments.py` provided in the example folder, which can serve as a guide on how to use these
+    datasets.
+
+    :param task: a string representing the task name; valid ones are T1, T2, T3, and T4
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quapy_data/ directory)
+    :param merge_T3: bool, if False (default), returns a generator of training collections, corresponding to natural
+        groups of reviews; if True, returns one single :class:`quapy.data.base.LabelledCollection` representing the
+        entire training set, as a concatenation of all the training collections
+    :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of
+        :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of
+        :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
+        that return a series of samples stored in a directory which are labelled by prevalence.
+    """
+
+    from quapy.data._lequa import load_vector_documents_2024, SamplesFromDir, LabelledCollectionsFromDir
+
+    assert task in LEQUA2024_TASKS, \
+        f'Unknown task {task}. Valid ones are {LEQUA2024_TASKS}'
+
+    if data_home is None:
+        data_home = get_quapy_home()
+
+    lequa_dir = data_home
+
+    LEQUA2024_ZENODO = 'https://zenodo.org/records/11661820'  # v3, last one with labels
+
+    URL_TRAINDEV=f'{LEQUA2024_ZENODO}/files/{task}.train_dev.zip'
+    URL_TEST=f'{LEQUA2024_ZENODO}/files/{task}.test.zip'
+    URL_TEST_PREV=f'{LEQUA2024_ZENODO}/files/{task}.test_prevalences.zip'
+
+    lequa_dir = join(data_home, 'lequa2024')
+    os.makedirs(lequa_dir, exist_ok=True)
+
+    def download_unzip_and_remove(unzipped_path, url):
+        tmp_path = join(lequa_dir, task + '_tmp.zip')
+        download_file_if_not_exists(url, tmp_path)
+        with zipfile.ZipFile(tmp_path) as file:
+            file.extractall(unzipped_path)
+        os.remove(tmp_path)
+
+    if not os.path.exists(join(lequa_dir, task)):
+        download_unzip_and_remove(lequa_dir, URL_TRAINDEV)
+        download_unzip_and_remove(lequa_dir, URL_TEST)
+        download_unzip_and_remove(lequa_dir, URL_TEST_PREV)
+
+    load_fn = load_vector_documents_2024
+
+    val_samples_path = join(lequa_dir, task, 'public', 'dev_samples')
+    val_true_prev_path = join(lequa_dir, task, 'public', 'dev_prevalences.txt')
+    val_gen = SamplesFromDir(val_samples_path, val_true_prev_path, load_fn=load_fn)
+
+    test_samples_path = join(lequa_dir, task, 'public', 'test_samples')
+    test_true_prev_path = join(lequa_dir, task, 'public', 'test_prevalences.txt')
+    test_gen = SamplesFromDir(test_samples_path, test_true_prev_path, load_fn=load_fn)
+
+    if task == 'T3':
+        training_samples_path = join(lequa_dir, task, 'public', 'training_samples')
+        training_true_prev_path = join(lequa_dir, task, 'public', 'training_prevalences.txt')
+        train_gen = LabelledCollectionsFromDir(training_samples_path, training_true_prev_path, load_fn=load_fn)
+        if merge_T3:
+            train = LabelledCollection.join(*list(train_gen()))
+            return train, val_gen, test_gen
+        else:
+            return train_gen, val_gen, test_gen
+    else:
+        tr_path = join(lequa_dir, task, 'public', 'training_data.txt')
+        train = LabelledCollection.load(tr_path, loader_func=load_fn)
+        return train, val_gen, test_gen

+
+
+
+

+[docs]
+def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None):
     """
     Loads the IFCB dataset for quantification from `Zenodo <https://zenodo.org/records/10036244>`_ (for more
     information on this dataset, please follow the zenodo link).
     This dataset is based on the data available publicly at
     `WHOI-Plankton repo <https://github.com/hsosik/WHOI-Plankton>`_.
-    The scripts for the processing are available at `P. González's repo <https://github.com/pglez82/IFCB_Zenodo>`_.
-    Basically, this is the IFCB dataset with precomputed features for testing quantification algorithms.
+    The dataset already comes with processed features.
+    The scripts used for the processing are available at `P. González's repo <https://github.com/pglez82/IFCB_Zenodo>`_.
 
     The datasets are downloaded only once, and stored for fast reuse.
 
@@ -833,7 +1110,7 @@
         i.e., a sampling protocol that returns a series of samples labelled by prevalence.
     """
 
-    from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split
+    from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split
 
     if data_home is None:
         data_home = get_quapy_home()
@@ -845,7 +1122,7 @@
     ifcb_dir = join(data_home, 'ifcb')
     os.makedirs(ifcb_dir, exist_ok=True)
 
-    def download_unzip_and_remove(unzipped_path, url):
+    def download_unzip_and_remove(unzipped_path, url):
         tmp_path = join(ifcb_dir, 'ifcb_tmp.zip')
         download_file_if_not_exists(url, tmp_path)
         with zipfile.ZipFile(tmp_path) as file:
@@ -871,7 +1148,7 @@
     if for_model_selection:
         # In this case, return 70% of training data as the training set and 30% as the test set
         samples = get_sample_list(train_samples_path)
-        train, test = generate_modelselection_split(samples, split=0.3)
+        train, test = generate_modelselection_split(samples, test_prop=0.3)
         train_gen = IFCBTrainSamplesFromDir(path_dir=train_samples_path, classes=classes, samples=train)
 
         # Test prevalence is computed from class labels
@@ -887,6 +1164,7 @@
         return train, test_gen
     else:
         return train_gen, test_gen

+
diff --git a/docs/build/html/_modules/quapy/data/preprocessing.html b/docs/build/html/_modules/quapy/data/preprocessing.html index a50aa64..dcfe434 100644 --- a/docs/build/html/_modules/quapy/data/preprocessing.html +++ b/docs/build/html/_modules/quapy/data/preprocessing.html @@ -1,22 +1,20 @@ + + - quapy.data.preprocessing — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.data.preprocessing — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - + + + + + @@ -42,7 +40,13 @@
@@ -70,21 +74,55 @@

Source code for quapy.data.preprocessing

-import numpy as np
-from scipy.sparse import spmatrix
-from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
-from sklearn.preprocessing import StandardScaler
-from tqdm import tqdm
+import numpy as np
+from scipy.sparse import spmatrix
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+
+import quapy as qp
+from quapy.data.base import Dataset
+from quapy.util import map_parallel
+from .base import LabelledCollection
+
+
+

+[docs]
+def instance_transformation(dataset:Dataset, transformer, inplace=False):
+    """
+    Transforms a :class:`quapy.data.base.Dataset` applying the `fit_transform` and `transform` functions
+    of a (sklearn's) transformer.
+
+    :param dataset: a :class:`quapy.data.base.Dataset` where the instances of training and test collections are
+        lists of str
+    :param transformer: TransformerMixin implementing `fit_transform` and `transform` functions
+    :param inplace: whether or not to apply the transformation inplace (True), or to a new copy (False, default)
+    :return: a new :class:`quapy.data.base.Dataset` with transformed instances (if inplace=False) or a reference to the
+        current Dataset (if inplace=True) where the instances have been transformed
+    """
+    training_transformed = transformer.fit_transform(*dataset.training.Xy)
+    test_transformed = transformer.transform(dataset.test.X)
+    orig_name = dataset.name
+
+    if inplace:
+        dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_)
+        dataset.test = LabelledCollection(test_transformed, dataset.test.labels, dataset.classes_)
+        if hasattr(transformer, 'vocabulary_'):
+            dataset.vocabulary = transformer.vocabulary_
+        return dataset
+    else:
+        training = LabelledCollection(training_transformed, dataset.training.labels.copy(), dataset.classes_)
+        test = LabelledCollection(test_transformed, dataset.test.labels.copy(), dataset.classes_)
+        vocab = None
+        if hasattr(transformer, 'vocabulary_'):
+            vocab = transformer.vocabulary_
+        return Dataset(training, test, vocabulary=vocab, name=orig_name)

 
-import quapy as qp
-from quapy.data.base import Dataset
-from quapy.util import map_parallel
-from .base import LabelledCollection
 
 
 

 [docs]
-def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs):
+def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs):
     """
     Transforms a :class:`quapy.data.base.Dataset` of textual instances into a :class:`quapy.data.base.Dataset` of
     tfidf weighted sparse vectors
@@ -103,24 +141,13 @@
     __check_type(dataset.test.instances, np.ndarray, str)
 
     vectorizer = TfidfVectorizer(min_df=min_df, sublinear_tf=sublinear_tf, **kwargs)
-    training_documents = vectorizer.fit_transform(dataset.training.instances)
-    test_documents = vectorizer.transform(dataset.test.instances)
-
-    if inplace:
-        dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.classes_)
-        dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.classes_)
-        dataset.vocabulary = vectorizer.vocabulary_
-        return dataset
-    else:
-        training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.classes_)
-        test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.classes_)
-        return Dataset(training, test, vectorizer.vocabulary_)

+    return instance_transformation(dataset, vectorizer, inplace)
[docs] -def reduce_columns(dataset: Dataset, min_df=5, inplace=False): +def reduce_columns(dataset: Dataset, min_df=5, inplace=False): """ Reduces the dimensionality of the instances, represented as a `csr_matrix` (or any subtype of `scipy.sparse.spmatrix`), of training and test documents by removing the columns of words which are not present @@ -138,7 +165,7 @@ __check_type(dataset.test.instances, spmatrix) assert dataset.training.instances.shape[1] == dataset.test.instances.shape[1], 'unaligned vector spaces' - def filter_by_occurrences(X, W): + def filter_by_occurrences(X, W): column_prevalence = np.asarray((X > 0).sum(axis=0)).flatten() take_columns = column_prevalence >= min_df X = X[:, take_columns] @@ -159,7 +186,7 @@
[docs] -def standardize(dataset: Dataset, inplace=False): +def standardize(dataset: Dataset, inplace=False): """ Standardizes the real-valued columns of a :class:`quapy.data.base.Dataset`. Standardization, aka z-scoring, of a variable `X` comes down to subtracting the average and normalizing by the @@ -170,19 +197,24 @@ :class:`quapy.data.base.Dataset` is to be returned :return: an instance of :class:`quapy.data.base.Dataset` """ - s = StandardScaler(copy=not inplace) - training = s.fit_transform(dataset.training.instances) - test = s.transform(dataset.test.instances) + s = StandardScaler() + train, test = dataset.train_test + std_train_X = s.fit_transform(train.X) + std_test_X = s.transform(test.X) if inplace: + dataset.training.instances = std_train_X + dataset.test.instances = std_test_X return dataset else: + training = LabelledCollection(std_train_X, train.labels, classes=train.classes_) + test = LabelledCollection(std_test_X, test.labels, classes=test.classes_) return Dataset(training, test, dataset.vocabulary, dataset.name)
[docs] -def index(dataset: Dataset, min_df=5, inplace=False, **kwargs): +def index(dataset: Dataset, min_df=5, inplace=False, **kwargs): """ Indexes the tokens of a textual :class:`quapy.data.base.Dataset` of string documents. To index a document means to replace each different token by a unique numerical index. @@ -219,7 +251,7 @@ -def __check_type(container, container_type=None, element_type=None): +def __check_type(container, container_type=None, element_type=None): if container_type: assert isinstance(container, container_type), \ f'unexpected type of container (expected {container_type}, found {type(container)})' @@ -230,7 +262,7 @@
[docs] -class IndexTransformer: +class IndexTransformer: """ This class implements a sklearn's-style transformer that indexes text as numerical ids for the tokens it contains, and that would be generated by sklearn's @@ -240,14 +272,14 @@ `CountVectorizer <https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html>`_ """ - def __init__(self, **kwargs): + def __init__(self, **kwargs): self.vect = CountVectorizer(**kwargs) self.unk = -1 # a valid index is assigned after fit self.pad = -2 # a valid index is assigned after fit
[docs] - def fit(self, X): + def fit(self, X): """ Fits the transformer, i.e., decides on the vocabulary, given a list of strings. @@ -264,7 +296,7 @@
[docs] - def transform(self, X, n_jobs=None): + def transform(self, X, n_jobs=None): """ Transforms the strings in `X` as lists of numerical ids @@ -279,13 +311,13 @@ - def _index(self, documents): + def _index(self, documents): vocab = self.vocabulary_.copy() return [[vocab.get(word, self.unk) for word in self.analyzer(doc)] for doc in tqdm(documents, 'indexing')]
[docs] - def fit_transform(self, X, n_jobs=None): + def fit_transform(self, X, n_jobs=None): """ Fits the transform on `X` and transforms it. @@ -298,7 +330,7 @@
[docs] - def vocabulary_size(self): + def vocabulary_size(self): """ Gets the length of the vocabulary according to which the document tokens have been indexed @@ -309,7 +341,7 @@
[docs] - def add_word(self, word, id=None, nogaps=True): + def add_word(self, word, id=None, nogaps=True): """ Adds a new token (regardless of whether it has been found in the text or not), with dedicated id. Useful to define special tokens for codifying unknown words, or padding tokens. diff --git a/docs/build/html/_modules/quapy/data/reader.html b/docs/build/html/_modules/quapy/data/reader.html index 4c9c163..827d348 100644 --- a/docs/build/html/_modules/quapy/data/reader.html +++ b/docs/build/html/_modules/quapy/data/reader.html @@ -1,22 +1,20 @@ + + - quapy.data.reader — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.data.reader — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - + + + + + @@ -42,7 +40,13 @@
@@ -70,14 +74,14 @@

Source code for quapy.data.reader

-import numpy as np
-from scipy.sparse import dok_matrix
-from tqdm import tqdm
+import numpy as np
+from scipy.sparse import dok_matrix
+from tqdm import tqdm
 
 
 

 [docs]
-def from_text(path, encoding='utf-8', verbose=1, class2int=True):
+def from_text(path, encoding='utf-8', verbose=1, class2int=True):
     """
     Reads a labelled colletion of documents.
     File fomart <0 or 1>\t<document>\n
@@ -111,7 +115,7 @@
 
 

 [docs]
-def from_sparse(path):
+def from_sparse(path):
     """
     Reads a labelled collection of real-valued instances expressed in sparse format
     File format <-1 or 0 or 1>[\s col(int):val(float)]\n
@@ -120,7 +124,7 @@
     :return: a `csr_matrix` containing the instances (rows), and a ndarray containing the labels
     """
 
-    def split_col_val(col_val):
+    def split_col_val(col_val):
         col, val = col_val.split(':')
         col, val = int(col) - 1, float(val)
         return col, val
@@ -148,7 +152,7 @@
 
 

 [docs]
-def from_csv(path, encoding='utf-8'):
+def from_csv(path, encoding='utf-8'):
     """
     Reads a csv file in which columns are separated by ','.
     File format <label>,<feat1>,<feat2>,...,<featn>\n
@@ -171,7 +175,7 @@
 
 

 [docs]
-def reindex_labels(y):
+def reindex_labels(y):
     """
     Re-indexes a list of labels as a list of indexes, and returns the classnames corresponding to the indexes.
     E.g.:
@@ -194,7 +198,7 @@
 
 

 [docs]
-def binarize(y, pos_class):
+def binarize(y, pos_class):
     """
     Binarizes a categorical array-like collection of labels towards the positive class `pos_class`. E.g.,:
 
diff --git a/docs/build/html/_modules/quapy/error.html b/docs/build/html/_modules/quapy/error.html
index 1613468..73a9a53 100644
--- a/docs/build/html/_modules/quapy/error.html
+++ b/docs/build/html/_modules/quapy/error.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.error — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.error — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -73,12 +76,14 @@
   
Source code for quapy.error
 """Implementation of error measures used for quantification"""
 
-import numpy as np
-from sklearn.metrics import f1_score
-import quapy as qp
+import numpy as np
+from sklearn.metrics import f1_score
+import quapy as qp
 
 
-
[docs]def from_name(err_name):
+

+[docs]
+def from_name(err_name):
     """Gets an error function from its name. E.g., `from_name("mae")`
     will return function :meth:`quapy.error.mae`
 
@@ -90,7 +95,10 @@
     return callable_error

 
 
-
[docs]def f1e(y_true, y_pred):
+
+

+[docs]
+def f1e(y_true, y_pred):
     """F1 error: simply computes the error in terms of macro :math:`F_1`, i.e.,
     :math:`1-F_1^M`, where :math:`F_1` is the harmonic mean of precision and recall,
     defined as :math:`\\frac{2tp}{2tp+fp+fn}`, with `tp`, `fp`, and `fn` standing
@@ -105,7 +113,10 @@
     return 1. - f1_score(y_true, y_pred, average='macro')

 
 
-
[docs]def acce(y_true, y_pred):
+
+

+[docs]
+def acce(y_true, y_pred):
     """Computes the error in terms of 1-accuracy. The accuracy is computed as
     :math:`\\frac{tp+tn}{tp+fp+fn+tn}`, with `tp`, `fp`, `fn`, and `tn` standing
     for true positives, false positives, false negatives, and true negatives,
@@ -118,89 +129,207 @@
     return 1. - (y_true == y_pred).mean()

 
 
-
[docs]def mae(prevs, prevs_hat):
+
+

+[docs]
+def mae(prevs_true, prevs_hat):
     """Computes the mean absolute error (see :meth:`quapy.error.ae`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :return: mean absolute error
     """
-    return ae(prevs, prevs_hat).mean()

+    return ae(prevs_true, prevs_hat).mean()

 
 
-
[docs]def ae(prevs, prevs_hat):
+
+

+[docs]
+def ae(prevs_true, prevs_hat):
     """Computes the absolute error between the two prevalence vectors.
      Absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`AE(p,\\hat{p})=\\frac{1}{|\\mathcal{Y}|}\\sum_{y\\in \\mathcal{Y}}|\\hat{p}(y)-p(y)|`,
      where :math:`\\mathcal{Y}` are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: absolute error
     """
-    assert prevs.shape == prevs_hat.shape, f'wrong shape {prevs.shape} vs. {prevs_hat.shape}'
-    return abs(prevs_hat - prevs).mean(axis=-1)

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape} vs. {prevs_hat.shape}'
+    return abs(prevs_hat - prevs_true).mean(axis=-1)

 
 
-
[docs]def nae(prevs, prevs_hat):
+
+

+[docs]
+def nae(prevs_true, prevs_hat):
     """Computes the normalized absolute error between the two prevalence vectors.
      Normalized absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`NAE(p,\\hat{p})=\\frac{AE(p,\\hat{p})}{z_{AE}}`,
      where :math:`z_{AE}=\\frac{2(1-\\min_{y\\in \\mathcal{Y}} p(y))}{|\\mathcal{Y}|}`, and :math:`\\mathcal{Y}`
      are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: normalized absolute error
     """
-    assert prevs.shape == prevs_hat.shape, f'wrong shape {prevs.shape} vs. {prevs_hat.shape}'
-    return abs(prevs_hat - prevs).sum(axis=-1)/(2*(1-prevs.min(axis=-1)))

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape} vs. {prevs_hat.shape}'
+    return abs(prevs_hat - prevs_true).sum(axis=-1)/(2 * (1 - prevs_true.min(axis=-1)))

 
 
-
[docs]def mnae(prevs, prevs_hat):
+
+

+[docs]
+def mnae(prevs_true, prevs_hat):
     """Computes the mean normalized absolute error (see :meth:`quapy.error.nae`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :return: mean normalized absolute error
     """
-    return nae(prevs, prevs_hat).mean()

+    return nae(prevs_true, prevs_hat).mean()

 
 
-
[docs]def mse(prevs, prevs_hat):
+
+

+[docs]
+def mse(prevs_true, prevs_hat):
     """Computes the mean squared error (see :meth:`quapy.error.se`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the
         true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the
         predicted prevalence values
     :return: mean squared error
     """
-    return se(prevs, prevs_hat).mean()

+    return se(prevs_true, prevs_hat).mean()

 
 
-
[docs]def se(prevs, prevs_hat):
+
+

+[docs]
+def se(prevs_true, prevs_hat):
     """Computes the squared error between the two prevalence vectors.
      Squared error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`SE(p,\\hat{p})=\\frac{1}{|\\mathcal{Y}|}\\sum_{y\\in \\mathcal{Y}}(\\hat{p}(y)-p(y))^2`,
      where
      :math:`\\mathcal{Y}` are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: absolute error
     """
-    return ((prevs_hat - prevs) ** 2).mean(axis=-1)

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    return ((prevs_hat - prevs_true) ** 2).mean(axis=-1)

 
 
-
[docs]def mkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def sre(prevs_true, prevs_hat, prevs_train, eps=0.):
+    """
+    Computes the squared ratio error between two prevalence vectors.
+    The squared ratio error between prevalence vectors :math:`p` and
+    :math:`\\hat{p}` with training prevalence :math:`p^{tr}` is:
+    :math:`SRE(p,\\hat{p},p^{tr})=\\frac{1}{|\\mathcal{Y}|}\\sum_{i \\in \\mathcal{Y}}(w_i-\\hat{w}_i)^2`,
+    where :math:`w_i=\\frac{p_i}{p^{tr}_i}`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :param prevs_train: array-like with the training prevalence values, or a single
+        prevalence vector when all comparisons refer to the same training set
+    :param eps: smoothing factor for the prevalence values (default 0, i.e., no smoothing)
+    :return: squared ratio error
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    prevs_train = np.asarray(prevs_train)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape=} vs {prevs_hat.shape=}'
+    assert prevs_true.shape[-1] == prevs_train.shape[-1], 'wrong shape for training prevalence'
+    if prevs_true.ndim == 2 and prevs_train.ndim == 1:
+        prevs_train = np.tile(prevs_train, reps=(prevs_true.shape[0], 1))
+    if eps > 0:
+        prevs_true = smooth(prevs_true, eps)
+        prevs_hat = smooth(prevs_hat, eps)
+        prevs_train = smooth(prevs_train, eps)
+
+    n_classes = prevs_true.shape[-1]
+    w = prevs_true / prevs_train
+    w_hat = prevs_hat / prevs_train
+    return (1. / n_classes) * np.sum((w - w_hat) ** 2., axis=-1)

+
+
+
+

+[docs]
+def msre(prevs_true, prevs_hat, prevs_train, eps=0.):
+    """
+    Computes the mean squared ratio error (see :meth:`quapy.error.sre`) across the sample pairs.
+
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape equal to prevs_true with the predicted prevalence values
+    :param prevs_train: array-like with the training prevalence values
+    :param eps: smoothing factor (default 0, i.e., no smoothing)
+    :return: mean squared ratio error
+    """
+    return np.mean(sre(prevs_true, prevs_hat, prevs_train, eps))

+
+
+
+

+[docs]
+def aitchisondist(prevs_true, prevs_hat):
+    """
+    Computes the Aitchison distance between two prevalence vectors.
+    The Aitchison distance between prevalence vectors :math:`p` and
+    :math:`\\hat{p}` is computed as
+    :math:`d_A(p,\\hat{p})=\\|\\mathrm{clr}(p)-\\mathrm{clr}(\\hat{p})\\|_2`,
+    where :math:`\\mathrm{clr}(p)_i=\\log p_i-\\frac{1}{|\\mathcal{Y}|}
+    \\sum_{j \\in \\mathcal{Y}} \\log p_j`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :return: Aitchison distance
+    """
+    from quapy.functional import CLRtransformation
+
+    clr = CLRtransformation()
+    return np.linalg.norm(clr(prevs_true) - clr(prevs_hat), axis=-1)

+
+
+
+

+[docs]
+def maitchisondist(prevs_true, prevs_hat):
+    """
+    Computes the mean Aitchison distance (see :meth:`quapy.error.aitchisondist`)
+    across the sample pairs, i.e.,
+    :math:`\\mathrm{mAitchisonDist}=\\frac{1}{n}\\sum_{i=1}^n
+    d_A(p_i,\\hat{p}_i)`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :return: mean Aitchison distance
+    """
+    return np.mean(aitchisondist(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def mkld(prevs_true, prevs_hat, eps=None):
     """Computes the mean Kullback-Leibler divergence (see :meth:`quapy.error.kld`) across the
     sample pairs. The distributions are smoothed using the `eps` factor
     (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -210,10 +339,13 @@
         (which has thus to be set beforehand).
     :return: mean Kullback-Leibler distribution
     """
-    return kld(prevs, prevs_hat, eps).mean()

+    return kld(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def kld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def kld(prevs_true, prevs_hat, eps=None):
     """Computes the Kullback-Leibler divergence between the two prevalence distributions.
      Kullback-Leibler divergence between two prevalence distributions :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -222,7 +354,7 @@
      where :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. KLD is not defined in cases in which the distributions contain
         zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the sample size.
@@ -231,17 +363,20 @@
     :return: Kullback-Leibler divergence between the two distributions
     """
     eps = __check_eps(eps)
-    smooth_prevs = prevs + eps
-    smooth_prevs_hat = prevs_hat + eps
+    smooth_prevs = smooth(prevs_true, eps)
+    smooth_prevs_hat = smooth(prevs_hat, eps)
     return (smooth_prevs*np.log(smooth_prevs/smooth_prevs_hat)).sum(axis=-1)

 
 
-
[docs]def mnkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mnkld(prevs_true, prevs_hat, eps=None):
     """Computes the mean Normalized Kullback-Leibler divergence (see :meth:`quapy.error.nkld`)
     across the sample pairs. The distributions are smoothed using the `eps` factor
     (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :param eps: smoothing factor. NKLD is not defined in cases in which the distributions contain
@@ -250,10 +385,13 @@
         (which has thus to be set beforehand).
     :return: mean Normalized Kullback-Leibler distribution
     """
-    return nkld(prevs, prevs_hat, eps).mean()

+    return nkld(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def nkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def nkld(prevs_true, prevs_hat, eps=None):
     """Computes the Normalized Kullback-Leibler divergence between the two prevalence distributions.
      Normalized Kullback-Leibler divergence between two prevalence distributions :math:`p` and
      :math:`\\hat{p}` is computed as
@@ -262,7 +400,7 @@
      :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. NKLD is not defined in cases in which the distributions
         contain zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the sample
@@ -270,16 +408,19 @@
         `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: Normalized Kullback-Leibler divergence between the two distributions
     """
-    ekld = np.exp(kld(prevs, prevs_hat, eps))
+    ekld = np.exp(kld(prevs_true, prevs_hat, eps))
     return 2. * ekld / (1 + ekld) - 1.

 
 
-
[docs]def mrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mrae(prevs_true, prevs_hat, eps=None):
     """Computes the mean relative absolute error (see :meth:`quapy.error.rae`) across
     the sample pairs. The distributions are smoothed using the `eps` factor (see
     :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -289,10 +430,13 @@
         the environment variable `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: mean relative absolute error
     """
-    return rae(prevs, prevs_hat, eps).mean()

+    return rae(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def rae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def rae(prevs_true, prevs_hat, eps=None):
     """Computes the absolute relative error between the two prevalence vectors.
      Relative absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -301,7 +445,7 @@
      where :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. `rae` is not defined in cases in which the true distribution
         contains zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the
@@ -310,12 +454,15 @@
     :return: relative absolute error
     """
     eps = __check_eps(eps)
-    prevs = smooth(prevs, eps)
+    prevs_true = smooth(prevs_true, eps)
     prevs_hat = smooth(prevs_hat, eps)
-    return (abs(prevs - prevs_hat) / prevs).mean(axis=-1)

+    return (abs(prevs_true - prevs_hat) / prevs_true).mean(axis=-1)

 
 
-
[docs]def nrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def nrae(prevs_true, prevs_hat, eps=None):
     """Computes the normalized absolute relative error between the two prevalence vectors.
      Relative absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -325,7 +472,7 @@
      and :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. `nrae` is not defined in cases in which the true distribution
         contains zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the
@@ -334,18 +481,21 @@
     :return: normalized relative absolute error
     """
     eps = __check_eps(eps)
-    prevs = smooth(prevs, eps)
+    prevs_true = smooth(prevs_true, eps)
     prevs_hat = smooth(prevs_hat, eps)
-    min_p = prevs.min(axis=-1)
-    return (abs(prevs - prevs_hat) / prevs).sum(axis=-1)/(prevs.shape[-1]-1+(1-min_p)/min_p)

+    min_p = prevs_true.min(axis=-1)
+    return (abs(prevs_true - prevs_hat) / prevs_true).sum(axis=-1)/(prevs_true.shape[-1] - 1 + (1 - min_p) / min_p)

 
 
-
[docs]def mnrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mnrae(prevs_true, prevs_hat, eps=None):
     """Computes the mean normalized relative absolute error (see :meth:`quapy.error.nrae`) across
     the sample pairs. The distributions are smoothed using the `eps` factor (see
     :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -355,10 +505,84 @@
         the environment variable `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: mean normalized relative absolute error
     """
-    return nrae(prevs, prevs_hat, eps).mean()

+    return nrae(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def smooth(prevs, eps):
+
+

+[docs]
+def nmd(prevs_true, prevs_hat):
+    """
+    Computes the Normalized Match Distance; which is the Normalized Distance multiplied by the factor
+    `1/(n-1)` to guarantee the measure ranges between 0 (best prediction) and 1 (worst prediction).
+
+    :param prevs_true: array-like of shape `(n_classes,)` or `(n_instances, n_classes)`  with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` or `(n_instances, n_classes)` with the predicted prevalence values
+    :return: float in [0,1]
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    n = prevs_true.shape[-1]
+    return (1./(n-1))*np.mean(match_distance(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def bias_binary(prevs_true, prevs_hat):
+    """
+    Computes the (positive) bias in a binary problem. The bias is simply the difference between the
+    predicted positive value and the true positive value, so that a positive such value indicates the
+    prediction has positive bias (i.e., it tends to overestimate) the true value, and negative otherwise.
+    :math:`bias(p,\\hat{p})=\\hat{p}_1-p_1`,
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
+        prevalence values
+    :return: binary bias
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape[-1] == 2 and prevs_true.shape[-1] == 2, f'bias_binary can only be applied to binary problems'
+    return prevs_hat[...,1]-prevs_true[...,1]

+
+
+
+

+[docs]
+def mean_bias_binary(prevs_true, prevs_hat):
+    """
+    Computes the mean of the (positive) bias in a binary problem.
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
+    :return: mean binary bias
+    """
+    return np.mean(bias_binary(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def md(prevs_true, prevs_hat, ERROR_TOL=1E-3):
+    """
+    Computes the Match Distance, under the assumption that the cost in mistaking class i with class i+1 is 1 in
+    all cases.
+
+    :param prevs_true: array-like of shape `(n_classes,)` or `(n_instances, n_classes)`  with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` or `(n_instances, n_classes)` with the predicted prevalence values
+    :return: float
+    """
+    P = np.cumsum(prevs_true, axis=-1)
+    P_hat = np.cumsum(prevs_hat, axis=-1)
+    assert np.all(np.isclose(P_hat[..., -1], 1.0, rtol=ERROR_TOL)), \
+        'arg error in match_distance: the array does not represent a valid distribution'
+    distances = np.abs(P-P_hat)
+    return distances[..., :-1].sum(axis=-1)

+
+
+
+

+[docs]
+def smooth(prevs, eps):
     """ Smooths a prevalence distribution with :math:`\\epsilon` (`eps`) as:
     :math:`\\underline{p}(y)=\\frac{\\epsilon+p(y)}{\\epsilon|\\mathcal{Y}|+
     \\displaystyle\\sum_{y\\in \\mathcal{Y}}p(y)}`
@@ -367,11 +591,13 @@
     :param eps: smoothing factor
     :return: array-like of shape `(n_classes,)` with the smoothed distribution
     """
+    prevs = np.asarray(prevs)
     n_classes = prevs.shape[-1]
     return (prevs + eps) / (eps * n_classes + 1)

 
 
-def __check_eps(eps=None):
+
+def __check_eps(eps=None):
     if eps is None:
         sample_size = qp.environ['SAMPLE_SIZE']
         if sample_size is None:
@@ -381,8 +607,8 @@
 
 
 CLASSIFICATION_ERROR = {f1e, acce}
-QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld}
-QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld}
+QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld, msre, maitchisondist}
+QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld, sre, aitchisondist}
 QUANTIFICATION_ERROR_SMOOTH = {kld, nkld, rae, nrae, mkld, mnkld, mrae}
 CLASSIFICATION_ERROR_NAMES = {func.__name__ for func in CLASSIFICATION_ERROR}
 QUANTIFICATION_ERROR_NAMES = {func.__name__ for func in QUANTIFICATION_ERROR}
@@ -394,6 +620,9 @@
 f1_error = f1e
 acc_error = acce
 mean_absolute_error = mae
+squared_ratio_error = sre
+dist_aitchison = aitchisondist
+mean_dist_aitchison = maitchisondist
 absolute_error = ae
 mean_relative_absolute_error = mrae
 relative_absolute_error = rae
@@ -401,6 +630,8 @@
 normalized_relative_absolute_error = nrae
 mean_normalized_absolute_error = mnae
 mean_normalized_relative_absolute_error = mnrae
+normalized_match_distance = nmd
+match_distance = md
 

 
            

diff --git a/docs/build/html/_modules/quapy/evaluation.html b/docs/build/html/_modules/quapy/evaluation.html
index 56d34a5..4907b5b 100644
--- a/docs/build/html/_modules/quapy/evaluation.html
+++ b/docs/build/html/_modules/quapy/evaluation.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.evaluation — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.evaluation — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,16 +74,18 @@
            

              
   
Source code for quapy.evaluation
-from typing import Union, Callable, Iterable
-import numpy as np
-from tqdm import tqdm
-import quapy as qp
-from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol, IterateProtocol
-from quapy.method.base import BaseQuantifier
-import pandas as pd
+from typing import Union, Callable, Iterable
+import numpy as np
+from tqdm import tqdm
+import quapy as qp
+from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol, IterateProtocol
+from quapy.method.base import BaseQuantifier
+import pandas as pd
 
 
-
[docs]def prediction(
+

+[docs]
+def prediction(
         model: BaseQuantifier,
         protocol: AbstractProtocol,
         aggr_speedup: Union[str, bool] = 'auto',
@@ -118,7 +123,7 @@
         # checks whether the prediction can be made more efficiently; this check consists in verifying if the model is
         # of type aggregative, if the protocol is based on LabelledCollection, and if the total number of documents to
         # classify using the protocol would exceed the number of test documents in the original collection
-        from quapy.method.aggregative import AggregativeQuantifier
+        from quapy.method.aggregative import AggregativeQuantifier
         if isinstance(model, AggregativeQuantifier) and isinstance(protocol, OnLabelledCollectionProtocol):
             if aggr_speedup == 'force':
                 apply_optimization = True
@@ -136,10 +141,11 @@
         protocol_with_predictions = protocol.on_preclassified_instances(pre_classified)
         return __prediction_helper(model.aggregate, protocol_with_predictions, verbose)
     else:
-        return __prediction_helper(model.quantify, protocol, verbose)

+        return __prediction_helper(model.predict, protocol, verbose)

 
 
-def __prediction_helper(quantification_fn, protocol: AbstractProtocol, verbose=False):
+
+def __prediction_helper(quantification_fn, protocol: AbstractProtocol, verbose=False):
     true_prevs, estim_prevs = [], []
     for sample_instances, sample_prev in tqdm(protocol(), total=protocol.total(), desc='predicting') if verbose else protocol():
         estim_prevs.append(quantification_fn(sample_instances))
@@ -151,7 +157,9 @@
     return true_prevs, estim_prevs
 
 
-
[docs]def evaluation_report(model: BaseQuantifier,
+

+[docs]
+def evaluation_report(model: BaseQuantifier,
                       protocol: AbstractProtocol,
                       error_metrics: Iterable[Union[str,Callable]] = 'mae',
                       aggr_speedup: Union[str, bool] = 'auto',
@@ -182,7 +190,8 @@
     return _prevalence_report(true_prevs, estim_prevs, error_metrics)

 
 
-def _prevalence_report(true_prevs, estim_prevs, error_metrics: Iterable[Union[str, Callable]] = 'mae'):
+
+def _prevalence_report(true_prevs, estim_prevs, error_metrics: Iterable[Union[str, Callable]] = 'mae'):
 
     if isinstance(error_metrics, str):
         error_metrics = [error_metrics]
@@ -203,7 +212,9 @@
     return df
 
 
-
[docs]def evaluate(
+

+[docs]
+def evaluate(
         model: BaseQuantifier,
         protocol: AbstractProtocol,
         error_metric: Union[str, Callable],
@@ -235,7 +246,10 @@
     return error_metric(true_prevs, estim_prevs)

 
 
-
[docs]def evaluate_on_samples(
+
+

+[docs]
+def evaluate_on_samples(
         model: BaseQuantifier,
         samples: Iterable[qp.data.LabelledCollection],
         error_metric: Union[str, Callable],
@@ -259,6 +273,7 @@
 
 
 
+
 

 
            

diff --git a/docs/build/html/_modules/quapy/functional.html b/docs/build/html/_modules/quapy/functional.html
index 1b02248..482f621 100644
--- a/docs/build/html/_modules/quapy/functional.html
+++ b/docs/build/html/_modules/quapy/functional.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
@@ -71,55 +74,94 @@

Source code for quapy.functional

-import itertools
-from collections import defaultdict
-from typing import Union, Callable
+import warnings
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from functools import lru_cache
+from typing import Literal, Union, Callable
+from numpy.typing import ArrayLike
 
-import scipy
-import numpy as np
+import scipy
+import numpy as np
+
+import quapy as qp
 
 
-
[docs]def prevalence_linspace(n_prevalences=21, repeats=1, smooth_limits_epsilon=0.01):
+# ------------------------------------------------------------------------------------------
+# General utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def classes_from_labels(labels):
     """
-    Produces an array of uniformly separated values of prevalence.
-    By default, produces an array of 21 prevalence values, with
-    step 0.05 and with the limits smoothed, i.e.:
-    [0.01, 0.05, 0.10, 0.15, ..., 0.90, 0.95, 0.99]
-
-    :param n_prevalences: the number of prevalence values to sample from the [0,1] interval (default 21)
-    :param repeats: number of times each prevalence is to be repeated (defaults to 1)
-    :param smooth_limits_epsilon: the quantity to add and subtract to the limits 0 and 1
-    :return: an array of uniformly separated prevalence values
+    Obtains a np.ndarray with the (sorted) classes
+    :param labels: array-like with the instances' labels
+    :return: a sorted np.ndarray with the class labels
     """
-    p = np.linspace(0., 1., num=n_prevalences, endpoint=True)
-    p[0] += smooth_limits_epsilon
-    p[-1] -= smooth_limits_epsilon
-    if p[0] > p[1]:
-        raise ValueError(f'the smoothing in the limits is greater than the prevalence step')
-    if repeats > 1:
-        p = np.repeat(p, repeats)
-    return p

+    classes = np.unique(labels)
+    classes.sort()
+    return classes

 
 
-
[docs]def prevalence_from_labels(labels, classes):
+
+

+[docs]
+def num_classes_from_labels(labels):
     """
-    Computed the prevalence values from a vector of labels.
+    Obtains the number of classes from an array-like of instance's labels
+    :param labels: array-like with the instances' labels
+    :return: int, the number of classes
+    """
+    return len(classes_from_labels(labels))

 
-    :param labels: array-like of shape `(n_instances)` with the label for each instance
+
+# ------------------------------------------------------------------------------------------
+# Counter utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def counts_from_labels(labels: ArrayLike, classes: ArrayLike) -> np.ndarray:
+    """
+    Computes the raw count values from a vector of labels.
+
+    :param labels: array-like of shape `(n_instances,)` with the label for each instance
     :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
         some classes have no examples.
-    :return: an ndarray of shape `(len(classes))` with the class prevalence values
+    :return: ndarray of shape `(len(classes),)` with the raw counts for each class, in the same order
+        as they appear in `classes`
     """
-    if labels.ndim != 1:
+    if np.asarray(labels).ndim != 1:
         raise ValueError(f'param labels does not seem to be a ndarray of label predictions')
     unique, counts = np.unique(labels, return_counts=True)
     by_class = defaultdict(lambda:0, dict(zip(unique, counts)))
-    prevalences = np.asarray([by_class[class_] for class_ in classes], dtype=float)
-    prevalences /= prevalences.sum()
+    counts = np.asarray([by_class[class_] for class_ in classes], dtype=int)
+    return counts

+
+
+
+

+[docs]
+def prevalence_from_labels(labels: ArrayLike, classes: ArrayLike):
+    """
+    Computes the prevalence values from a vector of labels.
+
+    :param labels: array-like of shape `(n_instances,)` with the label for each instance
+    :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
+        some classes have no examples.
+    :return: ndarray of shape `(len(classes),)` with the class proportions for each class, in the same order
+        as they appear in `classes`
+    """
+    counts = counts_from_labels(labels, classes)
+    prevalences = counts.astype(float) / np.sum(counts)
     return prevalences

 
 
-
[docs]def prevalence_from_probabilities(posteriors, binarize: bool = False):
+
+

+[docs]
+def prevalence_from_probabilities(posteriors: ArrayLike, binarize: bool = False):
     """
     Returns a vector of prevalence values from a matrix of posterior probabilities.
 
@@ -128,8 +170,9 @@
         converting the vectors of posterior probabilities into class indices, by taking the argmax).
     :return: array of shape `(n_classes,)` containing the prevalence values
     """
+    posteriors = np.asarray(posteriors)
     if posteriors.ndim != 2:
-        raise ValueError(f'param posteriors does not seem to be a ndarray of posteior probabilities')
+        raise ValueError(f'param posteriors does not seem to be a ndarray of posterior probabilities')
     if binarize:
         predictions = np.argmax(posteriors, axis=-1)
         return prevalence_from_labels(predictions, np.arange(posteriors.shape[1]))
@@ -139,25 +182,316 @@
         return prevalences

 
 
-
[docs]def as_binary_prevalence(positive_prevalence: Union[float, np.ndarray], clip_if_necessary=False):
+
+

+[docs]
+def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1) -> int:
+    """
+    Computes the number of valid prevalence combinations in the n_classes-dimensional simplex if `n_prevpoints` equally
+    distant prevalence values are generated and `n_repeats` repetitions are requested.
+    The computation comes down to calculating:
+
+    .. math::
+        \\binom{N+C-1}{C-1} \\times r
+
+    where `N` is `n_prevpoints-1`, i.e., the number of probability mass blocks to allocate, `C` is the number of
+    classes, and `r` is `n_repeats`. This solution comes from the
+    `Stars and Bars <https://brilliant.org/wiki/integer-equations-star-and-bars/>`_ problem.
+
+    :param int n_classes: number of classes
+    :param int n_prevpoints: number of prevalence points.
+    :param int n_repeats: number of repetitions for each prevalence combination
+    :return: The number of possible combinations. For example, if `n_classes`=2, `n_prevpoints`=5, `n_repeats`=1,
+        then the number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25],
+        and [1.0,0.0]
+    """
+    N = n_prevpoints-1
+    C = n_classes
+    r = n_repeats
+    return int(scipy.special.binom(N + C - 1, C - 1) * r)

+
+
+
+

+[docs]
+def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1) -> int:
+    """
+    Searches for the largest number of (equidistant) prevalence points to define for each of the `n_classes` classes so
+    that the number of valid prevalence values generated as combinations of prevalence points (points in a
+    `n_classes`-dimensional simplex) do not exceed combinations_budget.
+
+    :param int combinations_budget: maximum number of combinations allowed
+    :param int n_classes: number of classes
+    :param int n_repeats: number of repetitions for each prevalence combination
+    :return: the largest number of prevalence points that generate less than combinations_budget valid prevalences
+    """
+    assert n_classes > 0 and n_repeats > 0 and combinations_budget > 0, 'parameters must be positive integers'
+    n_prevpoints = 1
+    while True:
+        combinations = num_prevalence_combinations(n_prevpoints, n_classes, n_repeats)
+        if combinations > combinations_budget:
+            return n_prevpoints-1
+        else:
+            n_prevpoints += 1

+
+
+
+# ------------------------------------------------------------------------------------------
+# Prevalence vectors
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def as_binary_prevalence(positive_prevalence: Union[float, ArrayLike], clip_if_necessary: bool=False) -> np.ndarray:
     """
     Helper that, given a float representing the prevalence for the positive class, returns a np.ndarray of two
     values representing a binary distribution.
 
-    :param positive_prevalence: prevalence for the positive class
-    :param clip_if_necessary: if True, clips the value in [0,1] in order to guarantee the resulting distribution
+    :param positive_prevalence: float or array-like of floats with the prevalence for the positive class
+    :param bool clip_if_necessary: if True, clips the value in [0,1] in order to guarantee the resulting distribution
         is valid. If False, it then checks that the value is in the valid range, and raises an error if not.
     :return: np.ndarray of shape `(2,)`
     """
+    positive_prevalence = np.asarray(positive_prevalence, float)
     if clip_if_necessary:
         positive_prevalence = np.clip(positive_prevalence, 0, 1)
     else:
-        assert 0 <= positive_prevalence <= 1, 'the value provided is not a valid prevalence for the positive class'
+        assert np.logical_and(0 <= positive_prevalence, positive_prevalence  <= 1).all(), \
+            'the value provided is not a valid prevalence for the positive class'
     return np.asarray([1-positive_prevalence, positive_prevalence]).T

 
 
 
-
[docs]def HellingerDistance(P, Q) -> float:
+

+[docs]
+def strprev(prevalences: ArrayLike, prec: int=3) -> str:
+    """
+    Returns a string representation for a prevalence vector. E.g.,
+
+    >>> strprev([1/3, 2/3], prec=2)
+    >>> '[0.33, 0.67]'
+
+    :param prevalences: array-like of prevalence values
+    :param prec: int, indicates the float precision (number of decimal values to print)
+    :return: string
+    """
+    return '['+ ', '.join([f'{p:.{prec}f}' for p in prevalences]) + ']'

+
+
+
+

+[docs]
+def check_prevalence_vector(prevalences: ArrayLike, raise_exception: bool=False, tolerance: float=1e-08, aggr=True):
+    """
+    Checks that `prevalences` is a valid prevalence vector, i.e., it contains values in [0,1] and
+    the values sum up to 1. In other words, verifies that the `prevalences` vectors lies in the
+    probability simplex.
+
+    :param ArrayLike prevalences: the prevalence vector, or vectors, to check
+    :param bool raise_exception: whether to raise an exception if the vector (or any of the vectors) does
+        not lie in the simplex (default False)
+    :param float tolerance: error tolerance for the check `sum(prevalences) - 1 = 0`
+    :param bool aggr: if True (default) returns one single bool (True if all prevalence vectors are valid,
+        False otherwise), if False returns an array of bool, one for each prevalence vector
+    :return: a single bool True if `prevalences` is a vector of prevalence values that lies on the simplex,
+        or False otherwise; alternatively, if `prevalences` is a matrix of shape `(num_vectors, n_classes,)`
+        then it returns one such bool for each prevalence vector
+    """
+    prevalences = np.asarray(prevalences)
+
+    all_positive = prevalences>=0
+    if not all_positive.all():
+        if raise_exception:
+            raise ValueError('some prevalence vectors contain negative numbers; '
+                             'consider using the qp.functional.normalize_prevalence with '
+                             'any method from ["clip", "mapsimplex", "softmax"]')
+
+    all_close_1 = np.isclose(prevalences.sum(axis=-1), 1, atol=tolerance)
+    if not all_close_1.all():
+        if raise_exception:
+            raise ValueError('some prevalence vectors do not sum up to 1; '
+                             'consider using the qp.functional.normalize_prevalence with '
+                             'any method from ["l1", "clip", "mapsimplex", "softmax"]')
+
+    valid = np.logical_and(all_positive.all(axis=-1), all_close_1)
+    if aggr:
+        return valid.all()
+    else:
+        return valid

+
+
+
+

+[docs]
+def uniform_prevalence(n_classes):
+    """
+    Returns a vector representing the uniform distribution for `n_classes`
+
+    :param n_classes: number of classes
+    :return: np.ndarray with all values 1/n_classes
+    """
+    assert isinstance(n_classes, int) and n_classes>0, \
+        (f'param {n_classes} not understood; must be a positive integer representing the '
+         f'number of classes ')
+    return np.full(shape=n_classes, fill_value=1./n_classes)

+
+
+
+

+[docs]
+def normalize_prevalence(prevalences: ArrayLike, method='l1'):
+    """
+    Normalizes a vector or matrix of prevalence values. The normalization consists of applying a L1 normalization in
+    cases in which the prevalence values are not all-zeros, and to convert the prevalence values into `1/n_classes` in
+    cases in which all values are zero.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :param str method: indicates the normalization method to employ, options are:
+
+        * `l1`: applies L1 normalization (default); a 0 vector is mapped onto the uniform prevalence
+        * `clip`: clip values in [0,1] and then rescales so that the L1 norm is 1
+        * `mapsimplex`: projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `softmax`: applies softmax to all vectors
+        * `condsoftmax`: applies softmax only to invalid prevalence vectors
+
+    :return: a normalized vector or matrix of prevalence values
+    """
+    if method in ['none', None]:
+        return prevalences
+
+    prevalences = np.asarray(prevalences, dtype=float)
+
+    if method=='l1':
+        normalized = l1_norm(prevalences)
+        check_prevalence_vector(normalized, raise_exception=True)
+    elif method=='clip':
+        normalized = clip(prevalences)  # no need to check afterwards
+    elif method=='mapsimplex':
+        normalized = projection_simplex_sort(prevalences)
+    elif method=='softmax':
+        normalized = softmax(prevalences)
+    elif method=='condsoftmax':
+        normalized = condsoftmax(prevalences)
+    else:
+        raise ValueError(f'unknown {method=}, valid ones are ["l1", "clip", "mapsimplex", "softmax", "condsoftmax"]')
+
+    return normalized

+
+
+
+

+[docs]
+def l1_norm(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies L1 normalization to the `unnormalized_arr` so that it becomes a valid prevalence
+    vector. Zero vectors are mapped onto the uniform distribution. Raises an exception if
+    the resulting vectors are not valid distributions. This may happen when the original
+    prevalence vectors contain negative values. Use the `clip` normalization function
+    instead to avoid this possibility.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    n_classes = prevalences.shape[-1]
+    accum = prevalences.sum(axis=-1, keepdims=True)
+    prevalences = np.true_divide(prevalences, accum, where=accum > 0, out=None)
+    allzeros = accum.flatten() == 0
+    if any(allzeros):
+        if prevalences.ndim == 1:
+            prevalences = np.full(shape=n_classes, fill_value=1. / n_classes)
+        else:
+            prevalences[allzeros] = np.full(shape=n_classes, fill_value=1. / n_classes)
+    return prevalences

+
+
+
+

+[docs]
+def clip(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Clips the values in [0,1] and then applies the L1 normalization.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    clipped = np.clip(prevalences, 0, 1)
+    normalized = l1_norm(clipped)
+    return normalized

+
+
+
+

+[docs]
+def projection_simplex_sort(unnormalized_arr: ArrayLike) -> np.ndarray:
+    """Projects a point onto the probability simplex.
+
+    The code is adapted from Mathieu Blondel's BSD-licensed
+    `implementation <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+    (see function `projection_simplex_sort` in their repo) which is accompanying the paper
+
+    Mathieu Blondel, Akinori Fujino, and Naonori Ueda.
+    Large-scale Multiclass Support Vector Machine Training via Euclidean Projection onto the Simplex,
+    ICPR 2014, `URL <http://www.mblondel.org/publications/mblondel-icpr2014.pdf>`_
+
+    :param `unnormalized_arr`: point in n-dimensional space, shape `(n,)`
+    :return: projection of `unnormalized_arr` onto the (n-1)-dimensional probability simplex, shape `(n,)`
+    """
+    unnormalized_arr = np.asarray(unnormalized_arr)
+    n = len(unnormalized_arr)
+    u = np.sort(unnormalized_arr)[::-1]
+    cssv = np.cumsum(u) - 1.0
+    ind = np.arange(1, n + 1)
+    cond = u - cssv / ind > 0
+    rho = ind[cond][-1]
+    theta = cssv[cond][-1] / float(rho)
+    return np.maximum(unnormalized_arr - theta, 0)

+
+
+
+

+[docs]
+def softmax(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies the softmax function to all vectors even if the original vectors were valid distributions.
+    If you want to leave valid vectors untouched, use condsoftmax instead.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    normalized = scipy.special.softmax(prevalences, axis=-1)
+    return normalized

+
+
+
+

+[docs]
+def condsoftmax(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies the softmax function only to vectors that do not represent valid distributions.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    invalid_idx = ~ check_prevalence_vector(prevalences, aggr=False, raise_exception=False)
+    if isinstance(invalid_idx, np.bool_) and invalid_idx:
+        # only one vector
+        normalized = scipy.special.softmax(prevalences)
+    else:
+        prevalences = np.copy(prevalences)
+        prevalences[invalid_idx] = scipy.special.softmax(prevalences[invalid_idx], axis=-1)
+        normalized = prevalences
+    return normalized

+
+
+
+# ------------------------------------------------------------------------------------------
+# Divergences
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def HellingerDistance(P: np.ndarray, Q: np.ndarray) -> float:
     """
     Computes the Hellingher Distance (HD) between (discretized) distributions `P` and `Q`.
     The HD for two discrete distributions of `k` bins is defined as:
@@ -172,7 +506,10 @@
     return np.sqrt(np.sum((np.sqrt(P) - np.sqrt(Q))**2))

 
 
-
[docs]def TopsoeDistance(P, Q, epsilon=1e-20):
+
+

+[docs]
+def TopsoeDistance(P: np.ndarray, Q: np.ndarray, epsilon: float=1e-20):
     """
     Topsoe distance between two (discretized) distributions `P` and `Q`.
     The Topsoe distance for two discrete distributions of `k` bins is defined as:
@@ -186,9 +523,159 @@
     :return: float
     """
     return np.sum(P*np.log((2*P+epsilon)/(P+Q+epsilon)) + Q*np.log((2*Q+epsilon)/(P+Q+epsilon)))

-                  
 
-
[docs]def uniform_prevalence_sampling(n_classes, size=1):
+
+
+

+[docs]
+def get_divergence(divergence: Union[str, Callable]):
+    """
+    Guarantees that the divergence received as argument is a function. That is, if this argument is already
+    a callable, then it is returned, if it is instead a string, then tries to instantiate the corresponding
+    divergence from the string name.
+
+    :param divergence: callable or string indicating the name of the divergence function
+    :return: callable
+    """
+    if isinstance(divergence, str):
+        if divergence=='HD':
+            return HellingerDistance
+        elif divergence=='topsoe':
+            return TopsoeDistance
+        else:
+            raise ValueError(f'unknown divergence {divergence}')
+    elif callable(divergence):
+        return divergence
+    else:
+        raise ValueError(f'argument "divergence" not understood; use a str or a callable function')

+
+
+
+# ------------------------------------------------------------------------------------------
+# Solvers
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def argmin_prevalence(loss: Callable,
+                      n_classes: int,
+                      method: Literal["optim_minimize", "linear_search", "ternary_search"]='optim_minimize'):
+    """
+    Searches for the prevalence vector that minimizes a loss function.
+
+    :param loss: callable, the function to minimize
+    :param n_classes: int, number of classes
+    :param method: string indicating the search strategy. Possible values are::
+        'optim_minimize': uses scipy.optim
+        'linear_search': carries out a linear search for binary problems in the space [0, 0.01, 0.02, ..., 1]
+        'ternary_search': implements the ternary search (not yet implemented)
+    :return: np.ndarray, a prevalence vector
+    """
+    if method == 'optim_minimize':
+        return optim_minimize(loss, n_classes)
+    elif method == 'linear_search':
+        return linear_search(loss, n_classes)
+    elif method == 'ternary_search':
+        ternary_search(loss, n_classes)
+    else:
+        raise NotImplementedError()

+
+
+
+

+[docs]
+def optim_minimize(loss: Callable, n_classes: int, return_loss=False):
+    """
+    Searches for the optimal prevalence values, i.e., an `n_classes`-dimensional vector of the (`n_classes`-1)-simplex
+    that yields the smallest lost. This optimization is carried out by means of a constrained search using scipy's
+    SLSQP routine.
+
+    :param loss: (callable) the function to minimize
+    :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector
+    :param return_loss: bool, if True, returns also the value of the loss (default is False).
+    :return: (ndarray) the best prevalence vector found or a tuple which also contains the value of the loss
+        if return_loss=True
+    """
+    from scipy import optimize
+
+    # the initial point is set as the uniform distribution
+    uniform_distribution = uniform_prevalence(n_classes=n_classes)
+
+    # solutions are bounded to those contained in the unit-simplex
+    bounds = tuple((0, 1) for _ in range(n_classes))  # values in [0,1]
+    constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)})  # values summing up to 1
+    r = optimize.minimize(loss, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints)
+    
+    if return_loss:
+        return r.x, r.fun
+    else:
+        return r.x

+
+
+
+
+
+
+
+
+
+
+
+# ------------------------------------------------------------------------------------------
+# Sampling utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def prevalence_linspace(grid_points:int=21, repeats:int=1, smooth_limits_epsilon:float=0.01) -> np.ndarray:
+    """
+    Produces an array of uniformly separated values of prevalence.
+    By default, produces an array of 21 prevalence values, with
+    step 0.05 and with the limits smoothed, i.e.:
+    [0.01, 0.05, 0.10, 0.15, ..., 0.90, 0.95, 0.99]
+
+    :param grid_points: the number of prevalence values to sample from the [0,1] interval (default 21)
+    :param repeats: number of times each prevalence is to be repeated (defaults to 1)
+    :param smooth_limits_epsilon: the quantity to add and subtract to the limits 0 and 1
+    :return: an array of uniformly separated prevalence values
+    """
+    p = np.linspace(0., 1., num=grid_points, endpoint=True)
+    p[0] += smooth_limits_epsilon
+    p[-1] -= smooth_limits_epsilon
+    if p[0] > p[1]:
+        raise ValueError(f'the smoothing in the limits is greater than the prevalence step')
+    if repeats > 1:
+        p = np.repeat(p, repeats)
+    return p

+
+
+
+

+[docs]
+def uniform_prevalence_sampling(n_classes: int, size: int=1) -> np.ndarray:
     """
     Implements the `Kraemer algorithm <http://www.cs.cmu.edu/~nasmith/papers/smith+tromble.tr04.pdf>`_
     for sampling uniformly at random from the unit simplex. This implementation is adapted from this
@@ -214,24 +701,17 @@
     return u

 
 
+
 uniform_simplex_sampling = uniform_prevalence_sampling
 
 
-
[docs]def strprev(prevalences, prec=3):
-    """
-    Returns a string representation for a prevalence vector. E.g.,
+# ------------------------------------------------------------------------------------------
+# Adjustment
+# ------------------------------------------------------------------------------------------
 
-    >>> strprev([1/3, 2/3], prec=2)
-    >>> '[0.33, 0.67]'
-
-    :param prevalences: a vector of prevalence values
-    :param prec: float precision
-    :return: string
-    """
-    return '['+ ', '.join([f'{p:.{prec}f}' for p in prevalences]) + ']'

-
-
-
[docs]def adjusted_quantification(prevalence_estim, tpr, fpr, clip=True):
+

+[docs]
+def solve_adjustment_binary(prevalence_estim: ArrayLike, tpr: float, fpr: float, clip: bool=True):
     """
     Implements the adjustment of ACC and PACC for the binary case. The adjustment for a prevalence estimate of the
     positive class `p` comes down to computing:
@@ -239,10 +719,10 @@
     .. math::
         ACC(p) = \\frac{ p - fpr }{ tpr - fpr }
 
-    :param prevalence_estim: float, the estimated value for the positive class
-    :param tpr: float, the true positive rate of the classifier
-    :param fpr: float, the false positive rate of the classifier
-    :param clip: set to True (default) to clip values that might exceed the range [0,1]
+    :param float prevalence_estim: the estimated value for the positive class (`p` in the formula)
+    :param float tpr: the true positive rate of the classifier
+    :param float fpr: the false positive rate of the classifier
+    :param bool clip: set to True (default) to clip values that might exceed the range [0,1]
     :return: float, the adjusted count
     """
 
@@ -255,187 +735,216 @@
     return adjusted

 
 
-
[docs]def normalize_prevalence(prevalences):
-    """
-    Normalize a vector or matrix of prevalence values. The normalization consists of applying a L1 normalization in
-    cases in which the prevalence values are not all-zeros, and to convert the prevalence values into `1/n_classes` in
-    cases in which all values are zero.
 
-    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
-    :return: a normalized vector or matrix of prevalence values
+

+[docs]
+def solve_adjustment(
+    class_conditional_rates: np.ndarray,
+    unadjusted_counts: np.ndarray,
+    method: Literal["inversion", "invariant-ratio"],
+    solver: Literal["exact", "minimize", "exact-raise", "exact-cc"]) -> np.ndarray:
+    """
+    Function that tries to solve for :math:`p` the equation :math:`q = M p`, where :math:`q` is the vector of
+    `unadjusted counts` (as estimated, e.g., via classify and count) with :math:`q_i` an estimate of
+    :math:`P(\hat{Y}=y_i)`, and where :math:`M` is the matrix of `class-conditional rates` with :math:`M_{ij}` an
+    estimate of :math:`P(\hat{Y}=y_i|Y=y_j)`.
+
+    :param class_conditional_rates: array of shape `(n_classes, n_classes,)` with entry `(i,j)` being the estimate
+        of :math:`P(\hat{Y}=y_i|Y=y_j)`, that is, the probability that an instance that belongs to class :math:`y_j`
+        ends up being classified as belonging to class :math:`y_i`
+
+    :param unadjusted_counts: array of shape `(n_classes,)` containing the unadjusted prevalence values (e.g., as
+        estimated by CC or PCC)
+
+    :param str method: indicates the adjustment method to be used. Valid options are:
+
+        * `inversion`: tries to solve the equation :math:`q = M p` as :math:`p = M^{-1} q` where
+          :math:`M^{-1}` is the matrix inversion of :math:`M`. This inversion may not exist in
+          degenerated cases.
+        * `invariant-ratio`: invariant ratio estimator of `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation in :math:`M` with the normalization condition (i.e., that the sum of
+          all prevalence values must equal 1).
+
+    :param str solver: the method to use for solving the system of linear equations. Valid options are:
+
+        * `exact-raise`: tries to solve the system using matrix inversion. Raises an error if the matrix has rank
+          strictly lower than `n_classes`.
+        * `exact-cc`: if the matrix is not full rank, returns :math:`q` (i.e., the unadjusted counts) as the estimates
+        * `exact`: deprecated, defaults to 'exact-cc' (will be removed in future versions)
+        * `minimize`: minimizes a loss, so the solution always exists
     """
-    prevalences = np.asarray(prevalences)
-    n_classes = prevalences.shape[-1]
-    accum = prevalences.sum(axis=-1, keepdims=True)
-    prevalences = np.true_divide(prevalences, accum, where=accum>0)
-    allzeros = accum.flatten()==0
-    if any(allzeros):
-        if prevalences.ndim == 1:
-            prevalences = np.full(shape=n_classes, fill_value=1./n_classes)
-        else:
-            prevalences[accum.flatten()==0] = np.full(shape=n_classes, fill_value=1./n_classes)
-    return prevalences

+    if solver == "exact":
+        warnings.warn(
+            "The 'exact' solver is deprecated. Use 'exact-raise' or 'exact-cc'", DeprecationWarning, stacklevel=2)
+        solver = "exact-cc"
+
+    A = np.asarray(class_conditional_rates, dtype=float)
+    B = np.asarray(unadjusted_counts, dtype=float)
+
+    if method == "inversion":
+        pass  # We leave A and B unchanged
+    elif method == "invariant-ratio":
+        # Change the last equation to replace it with the normalization condition
+        A[-1, :] = 1.0
+        B[-1] = 1.0
+    else:
+        raise ValueError(f"unknown {method=}")
+
+    if solver == "minimize":
+        def loss(prev):
+            return np.linalg.norm(A @ prev - B)
+        return optim_minimize(loss, n_classes=A.shape[0])
+    elif solver in ["exact-raise", "exact-cc"]:
+        # Solvers based on matrix inversion, so we use try/except block
+        try:
+            return np.linalg.solve(A, B)
+        except np.linalg.LinAlgError:
+            # The matrix is not invertible.
+            # Depending on the solver, we either raise an error
+            # or return the classifier predictions without adjustment
+            if solver == "exact-raise":
+                raise
+            elif solver == "exact-cc":
+                return unadjusted_counts
+            else:
+                raise ValueError(f"Solver {solver} not known.")
+    else:
+        raise ValueError(f'unknown {solver=}')

 
 
-def __num_prevalence_combinations_depr(n_prevpoints:int, n_classes:int, n_repeats:int=1):
+
+# ------------------------------------------------------------------------------------------
+# Transformations from Compositional analysis
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+class CompositionalTransformation(ABC):
     """
-    Computes the number of prevalence combinations in the n_classes-dimensional simplex if `nprevpoints` equally distant
-    prevalence values are generated and `n_repeats` repetitions are requested.
-
-    :param n_classes: integer, number of classes
-    :param n_prevpoints: integer, number of prevalence points.
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: The number of possible combinations. For example, if n_classes=2, n_prevpoints=5, n_repeats=1, then the
-        number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25], and [1.0,0.0]
+    Abstract class of transformations for compositional data.
     """
-    __cache={}
-    def __f(nc,np):
-        if (nc,np) in __cache:  # cached result
-            return __cache[(nc,np)]
-        if nc==1:  # stop condition
-            return 1
-        else:  # recursive call
-            x = sum([__f(nc-1, np-i) for i in range(np)])
-            __cache[(nc,np)] = x
-            return x
-    return __f(n_classes, n_prevpoints) * n_repeats
+
+    EPSILON = 1e-12
+
+    @abstractmethod
+    def __call__(self, X):
+        ...
+
+

+[docs]
+    @abstractmethod
+    def inverse(self, Z):
+        ...

+

 
 
-
[docs]def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1):
+
+

+[docs]
+class CLRtransformation(CompositionalTransformation):
     """
-    Computes the number of valid prevalence combinations in the n_classes-dimensional simplex if `n_prevpoints` equally
-    distant prevalence values are generated and `n_repeats` repetitions are requested.
-    The computation comes down to calculating:
-
-    .. math::
-        \\binom{N+C-1}{C-1} \\times r
-
-    where `N` is `n_prevpoints-1`, i.e., the number of probability mass blocks to allocate, `C` is the number of
-    classes, and `r` is `n_repeats`. This solution comes from the
-    `Stars and Bars <https://brilliant.org/wiki/integer-equations-star-and-bars/>`_ problem.
-
-    :param n_classes: integer, number of classes
-    :param n_prevpoints: integer, number of prevalence points.
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: The number of possible combinations. For example, if n_classes=2, n_prevpoints=5, n_repeats=1, then the
-        number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25], and [1.0,0.0]
+    Centered log-ratio (CLR) transformation.
     """
-    N = n_prevpoints-1
-    C = n_classes
-    r = n_repeats
-    return int(scipy.special.binom(N + C - 1, C - 1) * r)

+
+    def __call__(self, X):
+        X = np.asarray(X)
+        X = qp.error.smooth(X, self.EPSILON)
+        geometric_mean = np.exp(np.mean(np.log(X), axis=-1, keepdims=True))
+        return np.log(X / geometric_mean)
+
+

+[docs]
+    def inverse(self, Z):
+        return scipy.special.softmax(Z, axis=-1)

+

 
 
-
[docs]def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1):
+
+

+[docs]
+class ILRtransformation(CompositionalTransformation):
     """
-    Searches for the largest number of (equidistant) prevalence points to define for each of the `n_classes` classes so
-    that the number of valid prevalence values generated as combinations of prevalence points (points in a
-    `n_classes`-dimensional simplex) do not exceed combinations_budget.
-
-    :param combinations_budget: integer, maximum number of combinations allowed
-    :param n_classes: integer, number of classes
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: the largest number of prevalence points that generate less than combinations_budget valid prevalences
+    Isometric log-ratio (ILR) transformation.
     """
-    assert n_classes > 0 and n_repeats > 0 and combinations_budget > 0, 'parameters must be positive integers'
-    n_prevpoints = 1
-    while True:
-        combinations = num_prevalence_combinations(n_prevpoints, n_classes, n_repeats)
-        if combinations > combinations_budget:
-            return n_prevpoints-1
-        else:
-            n_prevpoints += 1

+
+    def __call__(self, X):
+        X = np.asarray(X)
+        X = qp.error.smooth(X, self.EPSILON)
+        basis = self.get_V(X.shape[-1])
+        return np.log(X) @ basis.T
+
+

+[docs]
+    def inverse(self, Z):
+        Z = np.asarray(Z)
+        basis = self.get_V(Z.shape[-1] + 1)
+        logp = Z @ basis
+        p = np.exp(logp)
+        return p / np.sum(p, axis=-1, keepdims=True)

 
 
-
[docs]def check_prevalence_vector(p, raise_exception=False, toleranze=1e-08):
+

+[docs]
+    @lru_cache(maxsize=None)
+    def get_V(self, k):
+        helmert = np.zeros((k, k))
+        for i in range(1, k):
+            helmert[i, :i] = 1
+            helmert[i, i] = -i
+            helmert[i] = helmert[i] / np.sqrt(i * (i + 1))
+        return helmert[1:, :]

+

+
+
+
+

+[docs]
+def normalized_entropy(p):
     """
-    Checks that p is a valid prevalence vector, i.e., that it contains values in [0,1] and that the values sum up to 1.
+    Computes the normalized Shannon entropy of a prevalence vector.
 
-    :param p: the prevalence vector to check
-    :return: True if `p` is valid, False otherwise
+    :param p: array-like prevalence vector summing to 1
+    :return: float in [0,1]
     """
     p = np.asarray(p)
-    if not all(p>=0):
-        if raise_exception:
-            raise ValueError('the prevalence vector contains negative numbers')
-        return False
-    if not all(p<=1):
-        if raise_exception:
-            raise ValueError('the prevalence vector contains values >1')
-        return False
-    if not np.isclose(p.sum(), 1, atol=toleranze):
-        if raise_exception:
-            raise ValueError('the prevalence vector does not sum up to 1')
-        return False
-    return True

+    entropy = scipy.stats.entropy(p)
+    max_entropy = np.log(len(p))
+    return np.clip(entropy / max_entropy, 0, 1)

 
 
-
[docs]def get_divergence(divergence: Union[str, Callable]):
-    if isinstance(divergence, str):
-        if divergence=='HD':
-            return HellingerDistance
-        elif divergence=='topsoe':
-            return TopsoeDistance
-        else:
-            raise ValueError(f'unknown divergence {divergence}')
-    elif callable(divergence):
-        return divergence
-    else:
-        raise ValueError(f'argument "divergence" not understood; use a str or a callable function')

 
-
-
[docs]def argmin_prevalence(loss, n_classes, method='optim_minimize'):
-    if method == 'optim_minimize':
-        return optim_minimize(loss, n_classes)
-    elif method == 'linear_search':
-        return linear_search(loss, n_classes)
-    elif method == 'ternary_search':
-        raise NotImplementedError()
-    else:
-        raise NotImplementedError()

-
-
-
[docs]def optim_minimize(loss, n_classes):
+

+[docs]
+def antagonistic_prevalence(p, strength=1):
     """
-    Searches for the optimal prevalence values, i.e., an `n_classes`-dimensional vector of the (`n_classes`-1)-simplex
-    that yields the smallest lost. This optimization is carried out by means of a constrained search using scipy's
-    SLSQP routine.
+    Reflects a prevalence vector in ILR space and maps it back to the simplex.
 
-    :param loss: (callable) the function to minimize
-    :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector
-    :return: (ndarray) the best prevalence vector found
+    :param p: array-like prevalence vector
+    :param strength: reflection strength in ILR space
+    :return: prevalence vector in the simplex
     """
-    from scipy import optimize
-
-    # the initial point is set as the uniform distribution
-    uniform_distribution = np.full(fill_value=1 / n_classes, shape=(n_classes,))
-
-    # solutions are bounded to those contained in the unit-simplex
-    bounds = tuple((0, 1) for _ in range(n_classes))  # values in [0,1]
-    constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)})  # values summing up to 1
-    r = optimize.minimize(loss, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints)
-    return r.x

+    ilr = ILRtransformation()
+    z = ilr(p)
+    z_ant = -strength * z
+    return ilr.inverse(z_ant)

 
 
-
diff --git a/docs/build/html/_modules/quapy/method/_kdey.html b/docs/build/html/_modules/quapy/method/_kdey.html index 4e96e56..9d82912 100644 --- a/docs/build/html/_modules/quapy/method/_kdey.html +++ b/docs/build/html/_modules/quapy/method/_kdey.html @@ -1,23 +1,20 @@ + + - + - quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@
@@ -71,61 +74,85 @@

Source code for quapy.method._kdey

-from typing import Union
-import numpy as np
-from sklearn.base import BaseEstimator
-from sklearn.neighbors import KernelDensity
+import numpy as np
+from numbers import Real
+from sklearn.base import BaseEstimator
+from sklearn.neighbors import KernelDensity
 
-import quapy as qp
-from quapy.data import LabelledCollection
-from quapy.method.aggregative import AggregativeSoftQuantifier
-import quapy.functional as F
-
-from sklearn.metrics.pairwise import rbf_kernel
+import quapy as qp
+from quapy.method.aggregative import AggregativeSoftQuantifier
+import quapy.functional as F
+from scipy.special import logsumexp
+from sklearn.metrics.pairwise import rbf_kernel
 
 
-
[docs]class KDEBase:
+

+[docs]
+class KDEBase:
     """
     Common ancestor for KDE-based methods. Implements some common routines.
     """
 
     BANDWIDTH_METHOD = ['scott', 'silverman']
+    KERNELS = ['gaussian', 'aitchison', 'ilr']
 
     @classmethod
-    def _check_bandwidth(cls, bandwidth):
+    def _check_bandwidth(cls, bandwidth, kernel):
         """
         Checks that the bandwidth parameter is correct
 
         :param bandwidth: either a string (see BANDWIDTH_METHOD) or a float
-        :return: nothing, but raises an exception for invalid values
+        :return: the bandwidth if the check is passed, or raises an exception for invalid values
         """
-        assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, float), \
+        assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, Real), \
             f'invalid bandwidth, valid ones are {KDEBase.BANDWIDTH_METHOD} or float values'
-        if isinstance(bandwidth, float):
-            assert 0 < bandwidth < 1,  "the bandwith for KDEy should be in (0,1), since this method models the unit simplex"
+        if isinstance(bandwidth, Real):
+            bandwidth = float(bandwidth)
+        return bandwidth
 
-
[docs]    def get_kde_function(self, X, bandwidth):
+    @classmethod
+    def _check_kernel(cls, kernel):
+        assert kernel in KDEBase.KERNELS, f'unknown {kernel=}'
+        return kernel
+
+

+[docs]
+    def get_kde_function(self, X, bandwidth, kernel):
         """
         Wraps the KDE function from scikit-learn.
 
         :param X: data for which the density function is to be estimated
         :param bandwidth: the bandwidth of the kernel
+        :param kernel: the kernel family
         :return: a scikit-learn's KernelDensity object
         """
+        X = self.transform_posteriors(X, kernel)
+        bandwidth = self.effective_bandwidth(bandwidth, kernel)
         return KernelDensity(bandwidth=bandwidth).fit(X)

 
-
[docs]    def pdf(self, kde, X):
+
+

+[docs]
+    def pdf(self, kde, X, kernel, log_densities=False):
         """
         Wraps the density evalution of scikit-learn's KDE. Scikit-learn returns log-scores (s), so this
         function returns :math:`e^{s}`
 
         :param kde: a previously fit KDE function
         :param X: the data for which the density is to be estimated
+        :param kernel: the kernel family
         :return: np.ndarray with the densities
         """
-        return np.exp(kde.score_samples(X))

+        X = self.transform_posteriors(X, kernel)
+        log_density = kde.score_samples(X)
+        if log_densities:
+            return log_density
+        return np.exp(log_density)

 
-
[docs]    def get_mixture_components(self, X, y, n_classes, bandwidth):
+
+

+[docs]
+    def get_mixture_components(self, X, y, classes, bandwidth, kernel):
         """
         Returns an array containing the mixture components, i.e., the KDE functions for each class.
 
@@ -133,13 +160,72 @@
         :param y: the class labels
         :param n_classes: integer, the number of classes
         :param bandwidth: float, the bandwidth of the kernel
+        :param kernel: the kernel family
         :return: a list of KernelDensity objects, each fitted with the corresponding class-specific covariates
         """
-        return [self.get_kde_function(X[y == cat], bandwidth) for cat in range(n_classes)]

+        class_cond_X = []
+        for cat in classes:
+            selX = X[y==cat]
+            if selX.size==0:
+                raise ValueError(f'empty class {cat}')
+            class_cond_X.append(selX)
+        return [self.get_kde_function(X_cond_yi, bandwidth, kernel) for X_cond_yi in class_cond_X]

+
+
+

+[docs]
+    def transform_posteriors(self, X, kernel):
+        if kernel in {'aitchison', 'ilr'}:
+            X = self.shrink_posteriors(X)
+        if kernel == 'aitchison':
+            return self.clr_transform(X)
+        if kernel == 'ilr':
+            return self.ilr_transform(X)
+        return X

+
+
+

+[docs]
+    def shrink_posteriors(self, X):
+        shrinkage = getattr(self, 'shrinkage', 0.0)
+        if shrinkage <= 0:
+            return X
+        X = np.asarray(X)
+        n_classes = X.shape[-1]
+        uniform = np.full(n_classes, 1.0 / n_classes, dtype=X.dtype)
+        return (1.0 - shrinkage) * X + shrinkage * uniform

+
+
+

+[docs]
+    def effective_bandwidth(self, bandwidth, kernel):
+        shrinkage = getattr(self, 'shrinkage', 0.0)
+        if shrinkage > 0 and kernel in {'aitchison', 'ilr'} and isinstance(bandwidth, Real):
+            return (1.0 - shrinkage) * float(bandwidth)
+        return bandwidth

+
+
+

+[docs]
+    def clr_transform(self, X):
+        if not hasattr(self, 'clr'):
+            self.clr = F.CLRtransformation()
+        return self.clr(X)

+
+
+

+[docs]
+    def ilr_transform(self, X):
+        if not hasattr(self, 'ilr'):
+            self.ilr = F.ILRtransformation()
+        return self.ilr(X)

+

 
 
 
-
[docs]class KDEyML(AggregativeSoftQuantifier, KDEBase):
+

+[docs]
+class KDEyML(AggregativeSoftQuantifier, KDEBase):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Kullback-Leibler divergence (KLD) as
     the divergence measure to be minimized. This method was first proposed in the paper
@@ -165,32 +251,49 @@
 
     which corresponds to the maximum likelihood estimate.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
+    :param kernel: kernel of KDE, valid ones are in KDEBase.KERNELS
+    :param shrinkage: amount of shrinkage towards the uniform distribution to apply before
+        Aitchison/ILR transformations. Must be in ``[0,1)``.
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, bandwidth=0.1, n_jobs=None, random_state=None):
-        self._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1,
+                 kernel='gaussian', shrinkage=0.0, random_state=None):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel)
+        self.kernel = self._check_kernel(kernel)
+        assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)'
+        assert self.kernel != 'gaussian' or shrinkage == 0, \
+            'shrinkage is only supported for Aitchison/ILR kernels'
+        self.shrinkage = float(shrinkage)
         self.random_state=random_state
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        self.mix_densities = self.get_mixture_components(*classif_predictions.Xy, data.n_classes, self.bandwidth)
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self.mix_densities = self.get_mixture_components(
+            classif_predictions,
+            labels,
+            self.classes_,
+            self.bandwidth,
+            self.kernel,
+        )
         return self

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         """
         Searches for the mixture model parameter (the sought prevalence values) that maximizes the likelihood
         of the data (i.e., that minimizes the negative log-likelihood)
@@ -198,20 +301,35 @@
         :param posteriors: instances in the sample converted into posterior probabilities
         :return: a vector of class prevalence estimates
         """
-        np.random.RandomState(self.random_state)
-        epsilon = 1e-10
-        n_classes = len(self.mix_densities)
-        test_densities = [self.pdf(kde_i, posteriors) for kde_i in self.mix_densities]
+        with qp.util.temp_seed(self.random_state):
+            epsilon = 1e-12
+            n_classes = len(self.mix_densities)
+            if (self.kernel != 'gaussian' and n_classes >= 20) or n_classes >= 30:
+                test_log_densities = [
+                    self.pdf(kde_i, posteriors, self.kernel, log_densities=True)
+                    for kde_i in self.mix_densities
+                ]
 
-        def neg_loglikelihood(prev):
-            test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities))
-            test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
-            return  -np.sum(test_loglikelihood)
+                def neg_loglikelihood(prev):
+                    prev = qp.error.smooth(prev, eps=epsilon)
+                    test_loglikelihood = logsumexp(np.log(prev)[:, None] + test_log_densities, axis=0)
+                    return -np.sum(test_loglikelihood)
+            else:
+                test_densities = [self.pdf(kde_i, posteriors, self.kernel) for kde_i in self.mix_densities]
 
-        return F.optim_minimize(neg_loglikelihood, n_classes)

+                def neg_loglikelihood(prev):
+                    test_mixture_likelihood = prev @ test_densities
+                    test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
+                    return -np.sum(test_loglikelihood)
+
+            return F.optim_minimize(neg_loglikelihood, n_classes)

+

 
 
-
[docs]class KDEyHD(AggregativeSoftQuantifier, KDEBase):
+
+

+[docs]
+class KDEyHD(AggregativeSoftQuantifier, KDEBase):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the squared Hellinger Disntace (HD) as
     the divergence measure to be minimized. This method was first proposed in the paper
@@ -242,53 +360,59 @@
     where the datapoints (trials) :math:`x_1,\\ldots,x_t\\sim_{\\mathrm{iid}} r` with :math:`r`  the
     uniform distribution.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param montecarlo_trials: number of Monte Carlo trials (default 10000)
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, divergence: str='HD',
-                 bandwidth=0.1, n_jobs=None, random_state=None, montecarlo_trials=10000):
-        
-        self._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, divergence: str='HD',
+                 bandwidth=0.1, random_state=None, montecarlo_trials=10000):
+
+        super().__init__(classifier, fit_classifier, val_split)
         self.divergence = divergence
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian')
         self.random_state=random_state
         self.montecarlo_trials = montecarlo_trials
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        self.mix_densities = self.get_mixture_components(*classif_predictions.Xy, data.n_classes, self.bandwidth)
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self.mix_densities = self.get_mixture_components(
+            classif_predictions, labels, self.classes_, self.bandwidth, 'gaussian'
+        )
 
         N = self.montecarlo_trials
         rs = self.random_state
-        n = data.n_classes
+        n = len(self.classes_)
         self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.mix_densities])
-        self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.mix_densities])
+        self.reference_classwise_densities = np.asarray(
+            [self.pdf(kde_j, self.reference_samples, 'gaussian') for kde_j in self.mix_densities]
+        )
         self.reference_density = np.mean(self.reference_classwise_densities, axis=0)  # equiv. to (uniform @ self.reference_classwise_densities)
 
         return self

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         # we retain all n*N examples (sampled from a mixture with uniform parameter), and then
         # apply importance sampling (IS). In this version we compute D(p_alpha||q) with IS
         n_classes = len(self.mix_densities)
 
-        test_kde = self.get_kde_function(posteriors, self.bandwidth)
-        test_densities = self.pdf(test_kde, self.reference_samples)
+        test_kde = self.get_kde_function(posteriors, self.bandwidth, 'gaussian')
+        test_densities = self.pdf(test_kde, self.reference_samples, 'gaussian')
 
-        def f_squared_hellinger(u):
+        def f_squared_hellinger(u):
             return (np.sqrt(u)-1)**2
         
         # todo: this will fail when self.divergence is a callable, and is not the right place to do it anyway
@@ -304,15 +428,19 @@
         p_class = self.reference_classwise_densities + epsilon
         fracs = p_class/qs
 
-        def divergence(prev):
+        def divergence(prev):
             # ps / qs = (prev @ p_class) / qs = prev @ (p_class / qs) = prev @ fracs
             ps_div_qs = prev @ fracs
             return np.mean( f(ps_div_qs) * iw )
 
-        return F.optim_minimize(divergence, n_classes)

+        return F.optim_minimize(divergence, n_classes)

+

 
 
-
[docs]class KDEyCS(AggregativeSoftQuantifier):
+
+

+[docs]
+class KDEyCS(AggregativeSoftQuantifier):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Cauchy-Schwarz divergence (CS) as
     the divergence measure to be minimized. This method was first proposed in the paper
@@ -337,26 +465,25 @@
 
     The authors showed that this distribution matching admits a closed-form solution
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, bandwidth=0.1, n_jobs=None):
-        KDEBase._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian')
 
-
[docs]    def gram_matrix_mix_sum(self, X, Y=None):
+

+[docs]
+    def gram_matrix_mix_sum(self, X, Y=None):
         # this adapts the output of the rbf_kernel function (pairwise evaluations of Gaussian kernels k(x,y))
         # to contain pairwise evaluations of N(x|mu,Sigma1+Sigma2) with mu=y and Sigma1 and Sigma2 are 
         # two "scalar matrices" (h^2)*I each, so Sigma1+Sigma2 has scalar 2(h^2) (h is the bandwidth)
@@ -368,17 +495,20 @@
         gram = norm_factor * rbf_kernel(X, Y, gamma=gamma)
         return gram.sum()

 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
 
-        P, y = classif_predictions.Xy
-        n = data.n_classes
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+
+        P, y = classif_predictions, labels
+        n = len(self.classes_)
 
         assert all(sorted(np.unique(y)) == np.arange(n)), \
             'label name gaps not allowed in current implementation'
 
         # counts_inv keeps track of the relative weight of each datapoint within its class
         # (i.e., the weight in its KDE model)
-        counts_inv = 1 / (data.counts())
+        counts_inv = 1 / (F.counts_from_labels(y, classes=self.classes_))
 
         # tr_tr_sums corresponds to symbol \overline{B} in the paper
         tr_tr_sums = np.zeros(shape=(n,n), dtype=float)
@@ -399,7 +529,10 @@
         return self

 
 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         Ptr = self.Ptr
         Pte = posteriors
         y = self.ytr
@@ -419,16 +552,17 @@
         for i in range(n):
             tr_te_sums[i] = self.gram_matrix_mix_sum(Ptr[y==i], Pte) 
 
-        def divergence(alpha):
+        def divergence(alpha):
             # called \overline{r} in the paper
             alpha_ratio = alpha * self.counts_inv
 
-            # recal that tr_te_sums already accounts for the constant terms (1/Li)*(1/M)
+            # recall that tr_te_sums already accounts for the constant terms (1/Li)*(1/M)
             partA = -np.log((alpha_ratio @ tr_te_sums) * Minv)
             partB = 0.5 * np.log(alpha_ratio @ tr_tr_sums @ alpha_ratio)
             return partA + partB #+ partC
 
-        return F.optim_minimize(divergence, n)

+        return F.optim_minimize(divergence, n)

+
diff --git a/docs/build/html/_modules/quapy/method/_threshold_optim.html b/docs/build/html/_modules/quapy/method/_threshold_optim.html index 486aa61..6bb8003 100644 --- a/docs/build/html/_modules/quapy/method/_threshold_optim.html +++ b/docs/build/html/_modules/quapy/method/_threshold_optim.html @@ -1,23 +1,20 @@ + + - + - quapy.method._threshold_optim — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.method._threshold_optim — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@
@@ -71,17 +74,19 @@

Source code for quapy.method._threshold_optim

-from abc import abstractmethod
+from abc import abstractmethod
 
-import numpy as np
-from sklearn.base import BaseEstimator
-import quapy as qp
-import quapy.functional as F
-from quapy.data import LabelledCollection
-from quapy.method.aggregative import BinaryAggregativeQuantifier
+import numpy as np
+from sklearn.base import BaseEstimator
+import quapy as qp
+import quapy.functional as F
+from quapy.data import LabelledCollection
+from quapy.method.aggregative import BinaryAggregativeQuantifier
 
 
-
[docs]class ThresholdOptimization(BinaryAggregativeQuantifier):
+

+[docs]
+class ThresholdOptimization(BinaryAggregativeQuantifier):
     """
     Abstract class of Threshold Optimization variants for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -91,22 +96,29 @@
     that would allow for more true positives and many more false positives, on the grounds this
     would deliver larger denominators.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=None, n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    @abstractmethod
-    def condition(self, tpr, fpr) -> float:
+

+[docs]
+    @abstractmethod
+    def condition(self, tpr, fpr) -> float:
         """
         Implements the criterion according to which the threshold should be selected.
         This function should return the (float) score to be minimized.
@@ -117,7 +129,10 @@
         """
         ...

 
-
[docs]    def discard(self, tpr, fpr) -> bool:
+
+

+[docs]
+    def discard(self, tpr, fpr) -> bool:
         """
         Indicates whether a combination of tpr and fpr should be discarded
 
@@ -128,7 +143,8 @@
         return (tpr - fpr) == 0

 
 
-    def _eval_candidate_thresholds(self, decision_scores, y):
+
+    def _eval_candidate_thresholds(self, decision_scores, y):
         """
         Seeks for the best `tpr` and `fpr` according to the score obtained at different
         decision thresholds. The scoring function is implemented in function `_condition`.
@@ -163,7 +179,9 @@
 
         return candidates
 
-
[docs]    def aggregate_with_threshold(self, classif_predictions, tprs, fprs, thresholds):
+

+[docs]
+    def aggregate_with_threshold(self, classif_predictions, tprs, fprs, thresholds):
         # This function performs the adjusted count for given tpr, fpr, and threshold.
         # Note that, due to broadcasting, tprs, fprs, and thresholds could be arrays of length > 1
         prevs_estims = np.mean(classif_predictions[:, None] >= thresholds, axis=0)
@@ -171,35 +189,45 @@
         prevs_estims = F.as_binary_prevalence(prevs_estims, clip_if_necessary=True)
         return prevs_estims.squeeze()

 
-    def _compute_table(self, y, y_):
+
+    def _compute_table(self, y, y_):
         TP = np.logical_and(y == y_, y == self.pos_label).sum()
         FP = np.logical_and(y != y_, y == self.neg_label).sum()
         FN = np.logical_and(y != y_, y == self.pos_label).sum()
         TN = np.logical_and(y == y_, y == self.neg_label).sum()
         return TP, FP, FN, TN
 
-    def _compute_tpr(self, TP, FP):
+    def _compute_tpr(self, TP, FP):
         if TP + FP == 0:
             return 1
         return TP / (TP + FP)
 
-    def _compute_fpr(self, FP, TN):
+    def _compute_fpr(self, FP, TN):
         if FP + TN == 0:
             return 0
         return FP / (FP + TN)
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        decision_scores, y = classif_predictions.Xy
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        decision_scores, y = classif_predictions, labels
         # the standard behavior is to keep the best threshold only
         self.tpr, self.fpr, self.threshold = self._eval_candidate_thresholds(decision_scores, y)[0]
         return self

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
         # the standard behavior is to compute the adjusted count using the best threshold found
-        return self.aggregate_with_threshold(classif_predictions, self.tpr, self.fpr, self.threshold)

+        return self.aggregate_with_threshold(classif_predictions, self.tpr, self.fpr, self.threshold)

+

 
 
-
[docs]class T50(ThresholdOptimization):
+
+

+[docs]
+class T50(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -207,23 +235,34 @@
     for the threshold that makes `tpr` closest to 0.5.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
-        return abs(tpr - 0.5)

+

+[docs]
+    def condition(self, tpr, fpr) -> float:
+        return abs(tpr - 0.5)

+

 
 
-
[docs]class MAX(ThresholdOptimization):
+
+

+[docs]
+class MAX(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -231,24 +270,33 @@
     for the threshold that maximizes `tpr-fpr`.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
+

+[docs]
+    def condition(self, tpr, fpr) -> float:
         # MAX strives to maximize (tpr - fpr), which is equivalent to minimize (fpr - tpr)
-        return (fpr - tpr)

+        return (fpr - tpr)

+

 
 
-
[docs]class X(ThresholdOptimization):
+
+

+[docs]
+class X(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -256,23 +304,32 @@
     for the threshold that yields `tpr=1-fpr`.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
-        return abs(1 - (tpr + fpr))

+

+[docs]
+    def condition(self, tpr, fpr) -> float:
+        return abs(1 - (tpr + fpr))

+

 
 
-
[docs]class MS(ThresholdOptimization):
+
+

+[docs]
+class MS(ThresholdOptimization):
     """
     Median Sweep. Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -280,22 +337,30 @@
     class prevalence estimates for all decision thresholds and returns the median of them all.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def condition(self, tpr, fpr) -> float:
         return 1

 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        decision_scores, y = classif_predictions.Xy
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        decision_scores, y = classif_predictions, labels
         # keeps all candidates
         tprs_fprs_thresholds = self._eval_candidate_thresholds(decision_scores, y)
         self.tprs = tprs_fprs_thresholds[:, 0]
@@ -303,14 +368,21 @@
         self.thresholds = tprs_fprs_thresholds[:, 2]
         return self

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
         prevalences = self.aggregate_with_threshold(classif_predictions, self.tprs, self.fprs, self.thresholds)
         if prevalences.ndim==2:
             prevalences = np.median(prevalences, axis=0)
-        return prevalences

+        return prevalences

+

 
 
-
[docs]class MS2(MS):
+
+

+[docs]
+class MS2(MS):
     """
     Median Sweep 2. Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -319,19 +391,26 @@
     which `tpr-fpr>0.25`
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
 
-
[docs]    def discard(self, tpr, fpr) -> bool:
-        return (tpr-fpr) <= 0.25

+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def discard(self, tpr, fpr) -> bool:
+        return (tpr-fpr) <= 0.25

+

+
diff --git a/docs/build/html/_modules/quapy/method/aggregative.html b/docs/build/html/_modules/quapy/method/aggregative.html index 8311baa..307965d 100644 --- a/docs/build/html/_modules/quapy/method/aggregative.html +++ b/docs/build/html/_modules/quapy/method/aggregative.html @@ -1,23 +1,20 @@ + + - + - quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@
@@ -71,30 +74,42 @@

Source code for quapy.method.aggregative

-from abc import ABC, abstractmethod
-from copy import deepcopy
-from typing import Callable, Union
-import numpy as np
-from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
-from scipy import optimize
-from sklearn.base import BaseEstimator
-from sklearn.calibration import CalibratedClassifierCV
-from sklearn.metrics import confusion_matrix
-from sklearn.model_selection import cross_val_predict
+from abc import ABC, abstractmethod
+from argparse import ArgumentError
+from copy import deepcopy
+from typing import Callable, Literal, Union
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.exceptions import NotFittedError
+from sklearn.metrics import confusion_matrix
+from sklearn.model_selection import cross_val_predict, train_test_split
+from sklearn.utils.validation import check_is_fitted
+
+import quapy as qp
+import quapy.functional as F
+from quapy.functional import get_divergence
+from quapy.classification.svmperf import SVMperf
+from quapy.data import LabelledCollection
+from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric
+from quapy.method._helper import (
+    _get_abstention_calibrators,
+    _get_cvxpy,
+    _rlls_check_mode,
+    _rlls_joint_distribution,
+    _rlls_predicted_marginal,
+    _rlls_compute_3deltaC,
+    _rlls_compute_weights,
+)
 
-import quapy as qp
-import quapy.functional as F
-from quapy.functional import get_divergence
-from quapy.classification.calibration import NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration
-from quapy.classification.svmperf import SVMperf
-from quapy.data import LabelledCollection
-from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric
 
 
 # Abstract classes
 # ------------------------------------
 
-
[docs]class AggregativeQuantifier(BaseQuantifier, ABC):
+

+[docs]
+class AggregativeQuantifier(BaseQuantifier, ABC):
     """
     Abstract class for quantification methods that base their estimations on the aggregation of classification
     results. Aggregative quantifiers implement a pipeline that consists of generating classification predictions
@@ -108,22 +123,82 @@
 
     The method :meth:`quantify` comes with a default implementation based on :meth:`classify`
     and :meth:`aggregate`.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple `(X,y)` defining the specific set of data to use for validation. Set to
+        None when the method does not require any validation data, in order to avoid that some portion of
+        the training data be wasted.
     """
 
-    val_split_ = None
+    def __init__(self,
+                 classifier: Union[None,BaseEstimator],
+                 fit_classifier:bool=True,
+                 val_split:Union[int,float,tuple,None]=5):
 
-    @property
-    def val_split(self):
-        return self.val_split_
+        self.classifier = qp._get_classifier(classifier)
+        self.fit_classifier = fit_classifier
+        self.val_split = val_split
 
-    @val_split.setter
-    def val_split(self, val_split):
-        if isinstance(val_split, LabelledCollection):
-            print('warning: setting val_split with a LabelledCollection will be inefficient in'
-                  'model selection. Rather pass the LabelledCollection at fit time')
-        self.val_split_ = val_split
+        # basic type checks
+        assert hasattr(self.classifier, 'fit'), \
+            f'the classifier does not implement "fit"'
 
-    def _check_init_parameters(self):
+        assert isinstance(fit_classifier, bool), \
+            f'unexpected type for {fit_classifier=}; must be True or False'
+
+        # val_split is indicated as a number of folds for cross-validation
+        if isinstance(val_split, int):
+            assert val_split > 1, \
+                (f'when {val_split=} is indicated as an integer, it represents the number of folds in a kFCV '
+                 f'and must thus be >1')
+            if val_split==5 and not fit_classifier:
+                print(f'Warning: {val_split=} will be ignored when the classifier is already trained '
+                      f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} '
+                      f'to None to avoid this warning.')
+                self.val_split=None
+            if val_split!=5:
+                assert fit_classifier, (f'Parameter {val_split=} has been modified, but {fit_classifier=} '
+                                        f'indicates the classifier should not be retrained.')
+        # val_split is indicated as a fraction of validation instances
+        elif isinstance(val_split, float):
+            assert 0 < val_split < 1, \
+                (f'when {val_split=} is indicated as a float, it represents the fraction of training instances '
+                 f'to be used for validation, and must thus be in the range (0,1)')
+            assert fit_classifier, (f'when {val_split=} is indicated as a float (the fraction of training instances '
+                                    f'to be used for validation), the parameter {fit_classifier=} must be True')
+        # val_split is indicated as a validation collection (X,y)
+        elif isinstance(val_split, tuple):
+            assert len(val_split) == 2, \
+                (f'when {val_split=} is indicated as a tuple, it represents the collection (X,y) on which the '
+                 f'validation must be performed, but this seems to have different cardinality')
+        elif val_split is None:
+            pass
+        else:
+            raise ValueError(f'unexpected type for {val_split=}')
+
+        # classifier is fitted?
+        try:
+            check_is_fitted(self.classifier)
+            fitted = True
+        except NotFittedError:
+            fitted = False
+
+        # consistency checks: fit_classifier?
+        if self.fit_classifier:
+            if fitted:
+                raise RuntimeWarning(f'the classifier is already fitted, but {fit_classifier=} was requested')
+        else:
+            assert fitted, (f'{fit_classifier=} requires the classifier to be already trained, '
+                            f'but this does not seem to be')
+
+    def _check_init_parameters(self):
         """
         Implements any check to be performed in the parameters of the init method before undertaking
         the training of the quantifier. This is made as to allow for a quick execution stop when the
@@ -133,116 +208,103 @@
         """
         pass
 
-    def _check_non_empty_classes(self, data: LabelledCollection):
+    def _check_non_empty_classes(self, y):
         """
         Asserts all classes have positive instances.
 
-        :param data: LabelledCollection
+        :param labels: array-like of shape `(n_instances,)` with the label for each instance
+        :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
+            some classes have no examples.
         :return: Nothing. May raise an exception.
         """
-        sample_prevs = data.prevalence()
-        empty_classes = np.argwhere(sample_prevs==0).flatten()
-        if len(empty_classes)>0:
-            empty_class_names = data.classes_[empty_classes]
+        sample_prevs = F.prevalence_from_labels(y, self.classes_)
+        empty_classes = np.argwhere(sample_prevs == 0).flatten()
+        if len(empty_classes) > 0:
+            empty_class_names = self.classes_[empty_classes]
             raise ValueError(f'classes {empty_class_names} have no training examples')
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True, val_split=None):
+

+[docs]
+    def fit(self, X, y):
         """
-        Trains the aggregative quantifier. This comes down to training a classifier and an aggregation function.
+        Trains the aggregative quantifier. This comes down to training a classifier (if requested) and an
+        aggregation function.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
-        :param fit_classifier: whether to train the learner (default is True). Set to False if the
-            learner has been trained outside the quantifier.
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         :return: self
         """
         self._check_init_parameters()
-        classif_predictions = self.classifier_fit_predict(data, fit_classifier, predict_on=val_split)
-        self.aggregation_fit(classif_predictions, data)
+        classif_predictions, labels = self.classifier_fit_predict(X, y)
+        self.aggregation_fit(classif_predictions, labels)
         return self

 
-
[docs]    def classifier_fit_predict(self, data: LabelledCollection, fit_classifier=True, predict_on=None):
+
+

+[docs]
+    def classifier_fit_predict(self, X, y):
         """
         Trains the classifier if requested (`fit_classifier=True`) and generate the necessary predictions to
         train the aggregation function.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
-        :param fit_classifier: whether to train the learner (default is True). Set to False if the
-            learner has been trained outside the quantifier.
-        :param predict_on: specifies the set on which predictions need to be issued. This parameter can
-            be specified as None (default) to indicate no prediction is needed; a float in (0, 1) to
-            indicate the proportion of instances to be used for predictions (the remainder is used for
-            training); an integer >1 to indicate that the predictions must be generated via k-fold
-            cross-validation, using this integer as k; or the data sample itself on which to generate
-            the predictions.
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         """
-        assert isinstance(fit_classifier, bool), 'unexpected type for "fit_classifier", must be boolean'
+        self._check_classifier(adapt_if_necessary=self.fit_classifier)
 
-        self._check_classifier(adapt_if_necessary=(self._classifier_method() == 'predict_proba'))
+        # self._check_non_empty_classes(y)
 
-        if fit_classifier:
-            self._check_non_empty_classes(data)
-
-        if predict_on is None:
-            predict_on = self.val_split
-
-        if predict_on is None:
-            if fit_classifier:
-                self.classifier.fit(*data.Xy)
-            predictions = None
-        elif isinstance(predict_on, float):
-            if fit_classifier:
-                if not (0. < predict_on < 1.):
-                    raise ValueError(f'proportion {predict_on=} out of range, must be in (0,1)')
-                train, val = data.split_stratified(train_prop=(1 - predict_on))
-                self.classifier.fit(*train.Xy)
-                predictions = LabelledCollection(self.classify(val.X), val.y, classes=data.classes_)
+        predictions, labels = None, None
+        if isinstance(self.val_split, int):
+            assert self.fit_classifier, f'{self.__class__}: unexpected value for {self.fit_classifier=}'
+            num_folds = self.val_split
+            n_jobs = self.n_jobs if hasattr(self, 'n_jobs') else qp._get_njobs(None)
+            predictions = cross_val_predict(
+                self.classifier, X, y, cv=num_folds, n_jobs=n_jobs, method=self._classifier_method()
+            )
+            labels = y
+            self.classifier.fit(X, y)
+        elif isinstance(self.val_split, float):
+            assert self.fit_classifier, f'unexpected value for {self.fit_classifier=}'
+            train_prop = 1. - self.val_split
+            Xtr, Xval, ytr, yval = train_test_split(X, y, train_size=train_prop, stratify=y)
+            self.classifier.fit(Xtr, ytr)
+            predictions = self.classify(Xval)
+            labels = yval
+        elif isinstance(self.val_split, tuple):
+            Xval, yval = self.val_split
+            if self.fit_classifier:
+                self.classifier.fit(X, y)
+            predictions = self.classify(Xval)
+            labels = yval
+        elif self.val_split is None:
+            if self.fit_classifier:
+                self.classifier.fit(X, y)
+                predictions, labels = None, None
             else:
-                raise ValueError(f'wrong type for predict_on: since fit_classifier=False, '
-                                 f'the set on which predictions have to be issued must be '
-                                 f'explicitly indicated')
-
-        elif isinstance(predict_on, LabelledCollection):
-            if fit_classifier:
-                self.classifier.fit(*data.Xy)
-            predictions = LabelledCollection(self.classify(predict_on.X), predict_on.y, classes=predict_on.classes_)
-
-        elif isinstance(predict_on, int):
-            if fit_classifier:
-                if predict_on <= 1:
-                    raise ValueError(f'invalid value {predict_on} in fit. '
-                                     f'Specify a integer >1 for kFCV estimation.')
-                else:
-                    n_jobs = self.n_jobs if hasattr(self, 'n_jobs') else qp._get_njobs(None)
-                    predictions = cross_val_predict(
-                        self.classifier, *data.Xy, cv=predict_on, n_jobs=n_jobs, method=self._classifier_method())
-                    predictions = LabelledCollection(predictions, data.y, classes=data.classes_)
-                    self.classifier.fit(*data.Xy)
-            else:
-                raise ValueError(f'wrong type for predict_on: since fit_classifier=False, '
-                                 f'the set on which predictions have to be issued must be '
-                                 f'explicitly indicated')
-
+                predictions, labels = self.classify(X), y
         else:
-            raise ValueError(
-                f'error: param "predict_on" ({type(predict_on)}) not understood; '
-                f'use either a float indicating the split proportion, or a '
-                f'tuple (X,y) indicating the validation partition')
+            raise ValueError(f'unexpected type for {self.val_split=}')
 
-        return predictions

+        return predictions, labels

 
-
[docs]    @abstractmethod
-    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+
+

+[docs]
+    @abstractmethod
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Trains the aggregation function.
 
-        :param classif_predictions: a LabelledCollection containing the label predictions issued
-            by the classifier
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
+        :param classif_predictions: array-like with the classification predictions
+            (whatever the method :meth:`classify` returns)
+        :param labels: array-like with the true labels associated to each classifier prediction
         """
         ...

 
+
     @property
-    def classifier(self):
+    def classifier(self):
         """
         Gives access to the classifier
 
@@ -251,7 +313,7 @@
         return self.classifier_
 
     @classifier.setter
-    def classifier(self, classifier):
+    def classifier(self, classifier):
         """
         Setter for the classifier
 
@@ -259,18 +321,21 @@
         """
         self.classifier_ = classifier
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
         Provides the label predictions for the given instances. The predictions should respect the format expected by
         :meth:`aggregate`, e.g., posterior probabilities for probabilistic quantifiers, or crisp predictions for
         non-probabilistic quantifiers. The default one is "decision_function".
 
-        :param instances: array-like of shape `(n_instances, n_features,)`
-        :return: np.ndarray of shape `(n_instances,)` with label predictions
+        :param X: array-like of shape `(n_samples, n_features)`, the data instances
+        :return: np.ndarray of shape `(n_instances,)` with classifier predictions
         """
-        return getattr(self.classifier, self._classifier_method())(instances)

+        return getattr(self.classifier, self._classifier_method())(X)

 
-    def _classifier_method(self):
+
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. The default one is "decision_function".
 
@@ -278,56 +343,65 @@
         """
         return 'decision_function'
 
-    def _check_classifier(self, adapt_if_necessary=False):
+    def _check_classifier(self, adapt_if_necessary=False):
         """
         Guarantees that the underlying classifier implements the method required for issuing predictions, i.e.,
         the method indicated by the :meth:`_classifier_method`
 
-        :param adapt_if_necessary: if True, the method will try to comply with the required specifications
+        :param adapt_if_necessary: unused unless overridden
         """
         assert hasattr(self.classifier, self._classifier_method()), \
             f"the method does not implement the required {self._classifier_method()} method"
 
-
[docs]    def quantify(self, instances):
+

+[docs]
+    def predict(self, X):
         """
         Generate class prevalence estimates for the sample's instances by aggregating the label predictions generated
         by the classifier.
 
-        :param instances: array-like
+        :param X: array-like of shape `(n_samples, n_features)`, the data instances
         :return: `np.ndarray` of shape `(n_classes)` with class prevalence estimates.
         """
-        classif_predictions = self.classify(instances)
+        classif_predictions = self.classify(X)
         return self.aggregate(classif_predictions)

 
-
[docs]    @abstractmethod
-    def aggregate(self, classif_predictions: np.ndarray):
-        """
-        Implements the aggregation of label predictions.
 
-        :param classif_predictions: `np.ndarray` of label predictions
+

+[docs]
+    @abstractmethod
+    def aggregate(self, classif_predictions: np.ndarray):
+        """
+        Implements the aggregation of the classifier predictions.
+
+        :param classif_predictions: `np.ndarray` of classifier predictions
         :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
         """
         ...

 
+
     @property
-    def classes_(self):
+    def classes_(self):
         """
         Class labels, in the same order in which class prevalence values are to be computed.
         This default implementation actually returns the class labels of the learner.
 
-        :return: array-like
+        :return: array-like, the class labels
         """
         return self.classifier.classes_

 
 
-
[docs]class AggregativeCrispQuantifier(AggregativeQuantifier, ABC):
+
+

+[docs]
+class AggregativeCrispQuantifier(AggregativeQuantifier, ABC):
     """
-    Abstract class for quantification methods that base their estimations on the aggregation of crips decisions
+    Abstract class for quantification methods that base their estimations on the aggregation of crisp decisions
     as returned by a hard classifier. Aggregative crisp quantifiers thus extend Aggregative
     Quantifiers by implementing specifications about crisp predictions.
     """
 
-    def _classifier_method(self):
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. For crisp quantifiers, the method
         is 'predict', that returns an array of shape `(n_instances,)` of label predictions.
@@ -337,7 +411,10 @@
         return 'predict'

 
 
-
[docs]class AggregativeSoftQuantifier(AggregativeQuantifier, ABC):
+
+

+[docs]
+class AggregativeSoftQuantifier(AggregativeQuantifier, ABC):
     """
     Abstract class for quantification methods that base their estimations on the aggregation of posterior
     probabilities as returned by a probabilistic classifier.
@@ -345,7 +422,7 @@
     about soft predictions.
     """
 
-    def _classifier_method(self):
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. For probabilistic quantifiers, the method
         is 'predict_proba', that returns an array of shape `(n_instances, n_dimensions,)` with posterior
@@ -355,7 +432,7 @@
         """
         return 'predict_proba'
 
-    def _check_classifier(self, adapt_if_necessary=False):
+    def _check_classifier(self, adapt_if_necessary=False):
         """
         Guarantees that the underlying classifier implements the method indicated by the :meth:`_classifier_method`.
         In case it does not, the classifier is calibrated (by means of the Platt's calibration method implemented by
@@ -376,103 +453,234 @@
                                      f'fit_classifier is set to False')

 
 
-
[docs]class BinaryAggregativeQuantifier(AggregativeQuantifier, BinaryQuantifier):
-    
+
+

+[docs]
+class BinaryAggregativeQuantifier(AggregativeQuantifier, BinaryQuantifier):
+
     @property
-    def pos_label(self):
+    def pos_label(self):
         return self.classifier.classes_[1]
 
     @property
-    def neg_label(self):
+    def neg_label(self):
         return self.classifier.classes_[0]
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True, val_split=None):
-        self._check_binary(data, self.__class__.__name__)
-        return super().fit(data, fit_classifier, val_split)

+

+[docs]
+    def fit(self, X, y):
+        self._check_binary(y, self.__class__.__name__)
+        return super().fit(X, y)

+

+
 
 
 # Methods
 # ------------------------------------
-
[docs]class CC(AggregativeCrispQuantifier):
+

+[docs]
+class CC(AggregativeCrispQuantifier):
     """
     The most basic Quantification method. One that simply classifies all instances and counts how many have been
     attributed to each of the classes in order to compute class prevalence estimates.
 
     :param classifier: a sklearn's Estimator that generates a classifier
     """
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True):
+        super().__init__(classifier, fit_classifier, val_split=None)
 
-    def __init__(self, classifier: BaseEstimator):
-        self.classifier = classifier
-
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Nothing to do here!
 
-        :param classif_predictions: this is actually None
+        :param classif_predictions: unused
+        :param labels: unused
         """
         pass

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
         """
         Computes class prevalence estimates by counting the prevalence of each of the predicted labels.
 
-        :param classif_predictions: array-like with label predictions
+        :param classif_predictions: array-like with classifier predictions
         :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
         """
-        return F.prevalence_from_labels(classif_predictions, self.classes_)

+        return F.prevalence_from_labels(classif_predictions, self.classes_)

+

 
 
-
[docs]class ACC(AggregativeCrispQuantifier):
+
+

+[docs]
+class PCC(AggregativeSoftQuantifier):
+    """
+    `Probabilistic Classify & Count <https://ieeexplore.ieee.org/abstract/document/5694031>`_,
+    the probabilistic variant of CC that relies on the posterior probabilities returned by a probabilistic classifier.
+
+    :param classifier: a sklearn's Estimator that generates a classifier
+    """
+
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True, val_split=None):
+        super().__init__(classifier, fit_classifier, val_split=val_split)
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Nothing to do here!
+
+        :param classif_predictions: unused
+        :param labels: unused
+        """
+        pass

+
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
+        return F.prevalence_from_probabilities(classif_posteriors, binarize=False)

+

+
+
+
+

+[docs]
+class ACC(AggregativeCrispQuantifier):
     """
     `Adjusted Classify & Count <https://link.springer.com/article/10.1007/s10618-008-0097-y>`_,
     the "adjusted" variant of :class:`CC`, that corrects the predictions of CC
     according to the `misclassification rates`.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+    :param str method: adjustment method to be used:
+
+        * 'inversion': matrix inversion method based on the matrix equality :math:`P(C)=P(C|Y)P(Y)`,
+          which tries to invert :math:`P(C|Y)` matrix.
+        * 'invariant-ratio': invariant ratio estimator of `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation with the normalization condition.
+
+    :param str solver: indicates the method to use for solving the system of linear equations. Valid options are:
+
+        * 'exact-raise': tries to solve the system using matrix inversion. Raises an error if the matrix has rank
+          strictly less than `n_classes`.
+        * 'exact-cc': if the matrix is not of full rank, returns `p_c` as the estimates, which corresponds to
+          no adjustment (i.e., the classify and count method. See :class:`quapy.method.aggregative.CC`)
+        * 'exact': deprecated, defaults to 'exact-cc'
+        * 'minimize': minimizes the L2 norm of :math:`|Ax-B|`. This one generally works better, and is the
+          default parameter. More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of
+          Adjusted Classify and Count", on proceedings of the 2nd International Workshop on Learning to Quantify:
+          Methods and Applications (LQ 2022), ECML/PKDD 2022, Grenoble (France)
+          <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+
+    :param str norm: the method to use for normalization.
+
+        * `clip`, the values are clipped to the range [0,1] and then L1-normalized.
+        * `mapsimplex` projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `condsoftmax`, applies a softmax normalization only to prevalence vectors that lie outside the simplex
+
     :param n_jobs: number of parallel workers
-    :param solver: indicates the method to be used for obtaining the final estimates. The choice
-        'exact' comes down to solving the system of linear equations :math:`Ax=B` where `A` is a
-        matrix containing the class-conditional probabilities of the predictions (e.g., the tpr and fpr in 
-        binary) and `B` is the vector of prevalence values estimated via CC, as :math:`x=A^{-1}B`. This solution
-        might not exist for degenerated classifiers, in which case the method defaults to classify and count 
-        (i.e., does not attempt any adjustment).
-        Another option is to search for the prevalence vector that minimizes the L2 norm of :math:`|Ax-B|`. The latter
-        is achieved by indicating solver='minimize'. This one generally works better, and is the default parameter.
-        More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of Adjusted Classify and
-        Count", on proceedings of the 2nd International Workshop on Learning to Quantify: Methods and Applications
-        (LQ 2022), ECML/PKDD 2022, Grenoble (France) <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None, solver='minimize'):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier = True,
+            val_split = 5,
+            solver: Literal['minimize', 'exact', 'exact-raise', 'exact-cc'] = 'minimize',
+            method: Literal['inversion', 'invariant-ratio'] = 'inversion',
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+            n_jobs=None,
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
         self.solver = solver
+        self.method = method
+        self.norm = norm
 
-    def _check_init_parameters(self):
-        assert self.solver in ['exact', 'minimize'], "unknown solver; valid ones are 'exact', 'minimize'"
+    SOLVERS = ['exact', 'minimize', 'exact-raise', 'exact-cc']
+    METHODS = ['inversion', 'invariant-ratio']
+    NORMALIZATIONS = ['clip', 'mapsimplex', 'condsoftmax', None]
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    @classmethod
+    def newInvariantRatioEstimation(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, n_jobs=None):
+        """
+        Constructs a quantifier that implements the Invariant Ratio Estimator of
+        `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_. This amounts
+        to setting method to 'invariant-ratio' and clipping to 'project'.
+
+        :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+            the one indicated in `qp.environ['DEFAULT_CLS']`
+
+        :param fit_classifier: whether to train the learner (default is True). Set to False if the
+            learner has been trained outside the quantifier.
+
+        :param val_split: specifies the data used for generating classifier predictions. This specification
+            can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+            be extracted from the training set; or as an integer (default 5), indicating that the predictions
+            are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+            for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+        :param n_jobs: number of parallel workers
+
+        :return: an instance of ACC configured so that it implements the Invariant Ratio Estimator
+        """
+        return ACC(classifier, fit_classifier=fit_classifier, val_split=val_split, method='invariant-ratio', norm='mapsimplex', n_jobs=n_jobs)

+
+
+    def _check_init_parameters(self):
+        if self.solver not in ACC.SOLVERS:
+            raise ValueError(f"unknown solver; valid ones are {ACC.SOLVERS}")
+        if self.method not in ACC.METHODS:
+            raise ValueError(f"unknown method; valid ones are {ACC.METHODS}")
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Estimates the misclassification rates.
-
-        :param classif_predictions: classifier predictions with true labels
+        :param classif_predictions: array-like with the predicted labels
+        :param labels: array-like with the true labels associated to each predicted label
         """
-        pred_labels, true_labels = classif_predictions.Xy
-        self.cc = CC(self.classifier)
-        self.Pte_cond_estim_ = self.getPteCondEstim(self.classifier.classes_, true_labels, pred_labels)

+        true_labels = labels
+        pred_labels = classif_predictions
+        self.cc = CC(self.classifier, fit_classifier=False)
+        self.Pte_cond_estim_ = ACC.getPteCondEstim(self.classifier.classes_, true_labels, pred_labels)

 
-
[docs]    @classmethod
-    def getPteCondEstim(cls, classes, y, y_):
-        # estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
-        # document that belongs to yj ends up being classified as belonging to yi
+
+

+[docs]
+    @classmethod
+    def getPteCondEstim(cls, classes, y, y_):
+        """
+        Estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
+        document that belongs to yj ends up being classified as belonging to yi
+
+        :param classes: array-like with the class names
+        :param y: array-like with the true labels
+        :param y_: array-like with the estimated labels
+        :return: np.ndarray
+        """
         conf = confusion_matrix(y, y_, labels=classes).T
         conf = conf.astype(float)
         class_counts = conf.sum(axis=0)
@@ -483,124 +691,128 @@
                 conf[:, i] /= class_counts[i]
         return conf

 
-
[docs]    def aggregate(self, classif_predictions):
+
+

+[docs]
+    def aggregate(self, classif_predictions):
         prevs_estim = self.cc.aggregate(classif_predictions)
-        return ACC.solve_adjustment(self.Pte_cond_estim_, prevs_estim, solver=self.solver)

-
-
[docs]    @classmethod
-    def solve_adjustment(cls, PteCondEstim, prevs_estim, solver='exact'):
-        """
-        Solves the system linear system :math:`Ax = B` with :math:`A` = `PteCondEstim` and :math:`B` = `prevs_estim`
-
-        :param PteCondEstim: a `np.ndarray` of shape `(n_classes,n_classes,)` with entry `(i,j)` being the estimate
-            of :math:`P(y_i|y_j)`, that is, the probability that an instance that belongs to :math:`y_j` ends up being
-            classified as belonging to :math:`y_i`
-        :param prevs_estim: a `np.ndarray` of shape `(n_classes,)` with the class prevalence estimates
-        :param solver: indicates the method to use for solving the system of linear equations. Valid options are
-             'exact' (tries to solve the system --may fail if the misclassificatin matrix has rank < n_classes) or
-             'optim_minimize' (minimizes a norm --always exists). 
-        :return: an adjusted `np.ndarray` of shape `(n_classes,)` with the corrected class prevalence estimates
-        """
-
-        A = PteCondEstim
-        B = prevs_estim
-
-        if solver == 'exact':
-            # attempts an exact solution of the linear system (may fail)
-
-            try:
-                adjusted_prevs = np.linalg.solve(A, B)
-                adjusted_prevs = np.clip(adjusted_prevs, 0, 1)
-                adjusted_prevs /= adjusted_prevs.sum()
-            except np.linalg.LinAlgError:
-                adjusted_prevs = prevs_estim  # no way to adjust them!
-
-            return adjusted_prevs
-
-        elif solver == 'minimize':
-            # poses the problem as an optimization one, and tries to minimize the norm of the differences
-
-            def loss(prev):
-                return np.linalg.norm(A @ prev - B)
-
-            return F.optim_minimize(loss, n_classes=A.shape[0])

+        estimate = F.solve_adjustment(
+            class_conditional_rates=self.Pte_cond_estim_,
+            unadjusted_counts=prevs_estim,
+            solver=self.solver,
+            method=self.method,
+        )
+        return F.normalize_prevalence(estimate, method=self.norm)

+

 
 
-
[docs]class PCC(AggregativeSoftQuantifier):
-    """
-    `Probabilistic Classify & Count <https://ieeexplore.ieee.org/abstract/document/5694031>`_,
-    the probabilistic variant of CC that relies on the posterior probabilities returned by a probabilistic classifier.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    """
-
-    def __init__(self, classifier: BaseEstimator):
-        self.classifier = classifier
-
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        """
-        Nothing to do here!
-
-        :param classif_predictions: this is actually None
-        """
-        pass

-
-
[docs]    def aggregate(self, classif_posteriors):
-        return F.prevalence_from_probabilities(classif_posteriors, binarize=False)

-
-
-
[docs]class PACC(AggregativeSoftQuantifier):
+

+[docs]
+class PACC(AggregativeSoftQuantifier):
     """
     `Probabilistic Adjusted Classify & Count <https://ieeexplore.ieee.org/abstract/document/5694031>`_,
     the probabilistic variant of ACC that relies on the posterior probabilities returned by a probabilistic classifier.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`). Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
-    :param n_jobs: number of parallel workers
-    :param solver: indicates the method to be used for obtaining the final estimates. The choice
-        'exact' comes down to solving the system of linear equations :math:`Ax=B` where `A` is a
-        matrix containing the class-conditional probabilities of the predictions (e.g., the tpr and fpr in
-        binary) and `B` is the vector of prevalence values estimated via CC, as :math:`x=A^{-1}B`. This solution
-        might not exist for degenerated classifiers, in which case the method defaults to classify and count
-        (i.e., does not attempt any adjustment).
-        Another option is to search for the prevalence vector that minimizes the L2 norm of :math:`|Ax-B|`. The latter
-        is achieved by indicating solver='minimize'. This one generally works better, and is the default parameter.
-        More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of Adjusted Classify and
-        Count", on proceedings of the 2nd International Workshop on Learning to Quantify: Methods and Applications
-        (LQ 2022), ECML/PKDD 2022, Grenoble (France) <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
 
+    :param str method: adjustment method to be used:
+
+        * 'inversion': matrix inversion method based on the matrix equality :math:`P(C)=P(C|Y)P(Y)`,
+          which tries to invert `P(C|Y)` matrix.
+        * 'invariant-ratio': invariant ratio estimator of `Vaz et al. <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation with the normalization condition.
+
+    :param str solver: the method to use for solving the system of linear equations. Valid options are:
+
+        * 'exact-raise': tries to solve the system using matrix inversion.
+          Raises an error if the matrix has rank strictly less than `n_classes`.
+        * 'exact-cc': if the matrix is not of full rank, returns `p_c` as the estimates, which
+          corresponds to no adjustment (i.e., the classify and count method. See :class:`quapy.method.aggregative.CC`)
+        * 'exact': deprecated, defaults to 'exact-cc'
+        * 'minimize': minimizes the L2 norm of :math:`|Ax-B|`. This one generally works better, and is the
+          default parameter. More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions
+          of Adjusted Classify and Count", on proceedings of the 2nd International Workshop on Learning to
+          Quantify: Methods and Applications (LQ 2022), ECML/PKDD 2022, Grenoble (France)
+          <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+
+    :param str norm: the method to use for normalization.
+
+        * `clip`, the values are clipped to the range [0,1] and then L1-normalized.
+        * `mapsimplex` projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `condsoftmax`, applies a softmax normalization only to prevalence vectors that lie outside the simplex
+
+    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None, solver='minimize'):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier=True,
+            val_split=5,
+            solver: Literal['minimize', 'exact', 'exact-raise', 'exact-cc'] = 'minimize',
+            method: Literal['inversion', 'invariant-ratio'] = 'inversion',
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+            n_jobs=None
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
         self.solver = solver
+        self.method = method
+        self.norm = norm
 
-    def _check_init_parameters(self):
-        assert self.solver in ['exact', 'minimize'], "unknown solver; valid ones are 'exact', 'minimize'"
+    def _check_init_parameters(self):
+        if self.solver not in ACC.SOLVERS:
+            raise ValueError(f"unknown solver; valid ones are {ACC.SOLVERS}")
+        if self.method not in ACC.METHODS:
+            raise ValueError(f"unknown method; valid ones are {ACC.METHODS}")
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Estimates the misclassification rates
 
-        :param classif_predictions: classifier soft predictions with true labels
+        :param classif_predictions: array-like with posterior probabilities
+        :param labels: array-like with the true labels associated to each vector of posterior probabilities
         """
-        posteriors, true_labels = classif_predictions.Xy
-        self.pcc = PCC(self.classifier)
-        self.Pte_cond_estim_ = self.getPteCondEstim(self.classifier.classes_, true_labels, posteriors)

+        posteriors = classif_predictions
+        true_labels = labels
+        self.pcc = PCC(self.classifier, fit_classifier=False)
+        self.Pte_cond_estim_ = PACC.getPteCondEstim(self.classifier.classes_, true_labels, posteriors)

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         prevs_estim = self.pcc.aggregate(classif_posteriors)
-        return ACC.solve_adjustment(self.Pte_cond_estim_, prevs_estim, solver=self.solver)

 
-
[docs]    @classmethod
-    def getPteCondEstim(cls, classes, y, y_):
+        estimate = F.solve_adjustment(
+            class_conditional_rates=self.Pte_cond_estim_,
+            unadjusted_counts=prevs_estim,
+            solver=self.solver,
+            method=self.method,
+        )
+        return F.normalize_prevalence(estimate, method=self.norm)

+
+
+

+[docs]
+    @classmethod
+    def getPteCondEstim(cls, classes, y, y_):
         # estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
         # document that belongs to yj ends up being classified as belonging to yi
         n_classes = len(classes)
@@ -610,10 +822,125 @@
             if idx.any():
                 confusion[i] = y_[idx].mean(axis=0)
 
-        return confusion.T

+        return confusion.T

+

 
 
-
[docs]class EMQ(AggregativeSoftQuantifier):
+
+

+[docs]
+class RLLS(AggregativeSoftQuantifier):
+    """
+    `Regularized Learning for Domain Adaptation under Label Shifts
+    <https://arxiv.org/abs/1903.09734>`_, used here as an aggregative
+    quantifier.
+
+    This implementation ports the regularized weight-estimation component of
+    RLLS to QuaPy's aggregative interface. It estimates label-shift ratios from
+    validation posteriors and source labels, then rescales the source
+    prevalence to obtain target prevalence estimates.
+
+    This method relies on the optional `cvxpy` dependency.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case
+        the classifier is taken to be the one indicated in
+        `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set
+        to False if the learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier
+        predictions. This specification can be made as float in (0, 1)
+        indicating the proportion of stratified held-out validation set to be
+        extracted from the training set; or as an integer (default 5),
+        indicating that the predictions are to be generated in a `k`-fold
+        cross-validation manner; or as a tuple `(X, y)` defining the specific
+        set of data to use for validation. This method requires source
+        predictions and therefore needs `val_split` whenever
+        `fit_classifier=True`.
+    :param mode: whether source- and target-domain quantities are estimated
+        from posterior probabilities (`soft`, default) or from argmax
+        predictions (`hard`)
+    :param alpha: multiplicative factor for the regularization level (default
+        0.01)
+    :param delta: confidence parameter used in the finite-sample regularizer
+        (default 0.05)
+    :param clip_weights: if True, clips negative importance weights to zero
+        before converting them into prevalence estimates
+    :param norm: the normalization method passed to
+        :func:`quapy.functional.normalize_prevalence`
+    """
+
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier=True,
+            val_split=5,
+            mode: Literal['soft', 'hard'] = 'soft',
+            alpha: float = 0.01,
+            delta: float = 0.05,
+            clip_weights: bool = True,
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.mode = mode
+        self.alpha = alpha
+        self.delta = delta
+        self.clip_weights = clip_weights
+        self.norm = norm
+        self.last_w_ = None
+
+    def _check_init_parameters(self):
+        _get_cvxpy()
+        _rlls_check_mode(self.mode)
+        if not isinstance(self.alpha, (int, float)) or self.alpha < 0:
+            raise ValueError(f'expected a non-negative real value for alpha; found {self.alpha!r}')
+        if not isinstance(self.delta, (int, float)) or not (0 < self.delta < 1):
+            raise ValueError(f'expected delta to be in (0,1); found {self.delta!r}')
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
+        if self.fit_classifier and self.val_split is None:
+            raise ValueError(
+                'RLLS requires validation predictions for aggregation_fit; '
+                'please set val_split to an integer, float, or validation tuple'
+            )
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        if classif_predictions is None or labels is None:
+            raise ValueError('RLLS requires source posterior predictions and source labels')
+
+        self.train_prevalence_ = F.prevalence_from_labels(labels, classes=self.classes_)
+        self.C_zy_ = _rlls_joint_distribution(
+            classif_predictions,
+            labels,
+            self.classes_,
+            mode=self.mode,
+        )
+        self.pz_ = _rlls_predicted_marginal(classif_predictions, mode=self.mode)
+        self.rho_ = _rlls_compute_3deltaC(len(self.classes_), len(labels), self.delta)

+
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
+        qz = _rlls_predicted_marginal(classif_posteriors, mode=self.mode)
+        w = _rlls_compute_weights(
+            self.C_zy_,
+            qz,
+            self.pz_,
+            rho=self.alpha * self.rho_,
+            clip=self.clip_weights,
+        )
+        self.last_w_ = w
+        estimate = self.train_prevalence_ * w
+        return F.normalize_prevalence(estimate, method=self.norm)

+

+
+
+
+

+[docs]
+class EMQ(AggregativeSoftQuantifier):
     """
     `Expectation Maximization for Quantification <https://ieeexplore.ieee.org/abstract/document/6789744>`_ (EMQ),
     aka `Saerens-Latinne-Decaestecker` (SLD) algorithm.
@@ -626,108 +953,206 @@
     prevalence, an estimate of it obtained via k-fold cross validation (instead of the true training prevalence),
     and to recalibrate the posterior probabilities of the classifier.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: specifies the data used for generating classifier predictions. This specification
-        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
-        be extracted from the training set; or as an integer, indicating that the predictions
-        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`, default 5); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated. This hyperparameter is only meant to be used when the
-        heuristics are to be applied, i.e., if a recalibration is required. The default value is None (meaning
-        the recalibration is not required). In case this hyperparameter is set to a value other than None, but
-        the recalibration is not required (recalib=None), a warning message will be raised.
-    :param exact_train_prev: set to True (default) for using the true training prevalence as the initial observation;
-        set to False for computing the training prevalence as an estimate of it, i.e., as the expected
-        value of the posterior probabilities of the training instances.
-    :param recalib: a string indicating the method of recalibration.
-        Available choices include "nbvs" (No-Bias Vector Scaling), "bcts" (Bias-Corrected Temperature Scaling,
-        default), "ts" (Temperature Scaling), and "vs" (Vector Scaling). Default is None (no recalibration).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the classifier (default is True). Set to False if the
+        given classifier has already been trained.
+
+    :param val_split: specifies the data used for generating the classifier predictions on which the
+        aggregation function is to be trained. This specification can be made as float in (0, 1) indicating
+        the proportion of stratified held-out validation set to be extracted from the training set; or as
+        an integer (default 5), indicating that the predictions are to be generated in a `k`-fold
+        cross-validation manner (with this integer indicating the value for `k`); or as a tuple (X,y) defining
+        the specific set of data to use for validation. This hyperparameter is only meant to be used when
+        the heuristics are to be applied, i.e., if a calibration is required. The default value is None
+        (meaning the calibration is not required). In case this hyperparameter is set to a value other than
+        None, but the calibration is not required (calib=None), a warning message will be raised.
+
+    :param exact_train_prev: set to True (default) for using the true training prevalence as the initial
+        observation; set to False for computing the training prevalence as an estimate of it, i.e., as the
+        expected value of the posterior probabilities of the training instances.
+
+    :param calib: a string indicating the method of calibration.
+        Available choices include "nbvs" (No-Bias Vector Scaling), "bcts" (Bias-Corrected Temperature Scaling),
+        "ts" (Temperature Scaling), and "vs" (Vector Scaling). Default is None (no calibration).
+
+    :param on_calib_error: a string indicating the policy to follow in case the calibrator fails at runtime.
+        Options include "raise" (default), in which case a RuntimeException is raised; and "backup", in which
+        case the calibrator is silently skipped.
+
     :param n_jobs: number of parallel workers. Only used for recalibrating the classifier if `val_split` is set to
         an integer `k` --the number of folds.
     """
 
     MAX_ITER = 1000
     EPSILON = 1e-4
+    ON_CALIB_ERROR_VALUES = ['raise', 'backup']
+    CALIB_OPTIONS = [None, 'nbvs', 'bcts', 'ts', 'vs']
 
-    def __init__(self, classifier: BaseEstimator, val_split=None, exact_train_prev=True, recalib=None, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=None, exact_train_prev=True,
+                 calib=None, on_calib_error='raise', n_jobs=None):
+
+        assert calib in EMQ.CALIB_OPTIONS, \
+            f'invalid value for {calib=}; valid ones are {EMQ.CALIB_OPTIONS}'
+        assert on_calib_error in EMQ.ON_CALIB_ERROR_VALUES, \
+            f'invalid value for {on_calib_error=}; valid ones are {EMQ.ON_CALIB_ERROR_VALUES}'
+
+        super().__init__(classifier, fit_classifier, val_split)
         self.exact_train_prev = exact_train_prev
-        self.recalib = recalib
+        self.calib = calib
+        self.on_calib_error = on_calib_error
         self.n_jobs = n_jobs
 
-
[docs]    @classmethod
-    def EMQ_BCTS(cls, classifier: BaseEstimator, n_jobs=None):
+

+[docs]
+    @classmethod
+    def EMQ_BCTS(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, on_calib_error="raise", n_jobs=None):
         """
         Constructs an instance of EMQ using the best configuration found in the `Alexandari et al. paper
         <http://proceedings.mlr.press/v119/alexandari20a.html>`_, i.e., one that relies on Bias-Corrected Temperature
-        Scaling (BCTS) as a recalibration function, and that uses an estimate of the training prevalence instead of
+        Scaling (BCTS) as a calibration function, and that uses an estimate of the training prevalence instead of
         the true training prevalence.
 
-        :param classifier: a sklearn's Estimator that generates a classifier
-        :param n_jobs: number of parallel workers.
+        :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+            the one indicated in `qp.environ['DEFAULT_CLS']`
+
+        :param fit_classifier: whether to train the learner (default is True). Set to False if the
+            learner has been trained outside the quantifier.
+
+        :param val_split: specifies the data used for generating classifier predictions. This specification
+            can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+            be extracted from the training set; or as an integer (default 5), indicating that the predictions
+            are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+            for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+        :param on_calib_error: a string indicating the policy to follow in case the calibrator fails at runtime.
+            Options include "raise" (default), in which case a RuntimeException is raised; and "backup", in which
+            case the calibrator is silently skipped.
+
+        :param n_jobs: number of parallel workers. Only used for recalibrating the classifier if `val_split` is set to
+            an integer `k` --the number of folds.
+
         :return: An instance of EMQ with BCTS
         """
-        return EMQ(classifier, val_split=5, exact_train_prev=False, recalib='bcts', n_jobs=n_jobs)

+        return EMQ(classifier, fit_classifier=fit_classifier, val_split=val_split, exact_train_prev=False,
+                   calib='bcts', on_calib_error=on_calib_error, n_jobs=n_jobs)

 
-    def _check_init_parameters(self):
+
+    def _check_init_parameters(self):
         if self.val_split is not None:
-            if self.exact_train_prev and self.recalib is None:
+            if self.exact_train_prev and self.calib is None:
                 raise RuntimeWarning(f'The parameter {self.val_split=} was specified for EMQ, while the parameters '
-                      f'{self.exact_train_prev=} and {self.recalib=}. This has no effect and causes an unnecessary '
-                      f'overload.')
+                                     f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an '
+                                     f'unnecessary overload.')
         else:
-            if self.recalib is not None:
-                print(f'[warning] The parameter {self.recalib=} requires the val_split be different from None. '
+            if self.calib is not None:
+                print(f'[warning] The parameter {self.calib=} requires the val_split be different from None. '
                       f'This parameter will be set to 5. To avoid this warning, set this value to a float value '
                       f'indicating the proportion of training data to be used as validation, or to an integer '
                       f'indicating the number of folds for kFCV.')
-                self.val_split=5
+                self.val_split = 5
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
-        Provides the posterior probabilities for the given instances. If the classifier was required
-        to be recalibrated, then these posteriors are recalibrated accordingly.
+        Provides the posterior probabilities for the given instances. The calibration function, if required,
+        has no effect in this step, and is only involved in the aggregate method.
 
-        :param instances: array-like of shape `(n_instances, n_dimensions,)`
+        :param X: array-like of shape `(n_instances, n_dimensions,)`
         :return: np.ndarray of shape `(n_instances, n_classes,)` with posterior probabilities
         """
-        posteriors = self.classifier.predict_proba(instances)
+        return self.classifier.predict_proba(X)

+
+
+

+[docs]
+    def classifier_fit_predict(self, X, y):
+        classif_predictions = super().classifier_fit_predict(X, y)
+        self.train_prevalence = F.prevalence_from_labels(y, classes=self.classes_)
+        return classif_predictions

+
+
+    def _fit_calibration(self, calibrator, P, y):
+        n_classes = len(self.classes_)
+
+        if not np.issubdtype(y.dtype, np.number):
+            y = np.searchsorted(self.classes_, y)
+
+        try:
+            self.calibration_function = calibrator(P, np.eye(n_classes)[y], posterior_supplied=True)
+        except Exception as e:
+            if self.on_calib_error == 'raise':
+                raise RuntimeError(f'calibration {self.calib} failed at fit time: {e}')
+            elif self.on_calib_error == 'backup':
+                self.calibration_function = lambda P: P
+
+    def _calibrate_if_requested(self, uncalib_posteriors):
         if hasattr(self, 'calibration_function') and self.calibration_function is not None:
-            posteriors = self.calibration_function(posteriors)
-        return posteriors

+            try:
+                calib_posteriors = self.calibration_function(uncalib_posteriors)
+            except Exception as e:
+                if self.on_calib_error == 'raise':
+                    raise RuntimeError(f'calibration {self.calib} failed at predict time: {e}')
+                elif self.on_calib_error == 'backup':
+                    calib_posteriors = uncalib_posteriors
+                else:
+                    raise ValueError(f'unexpected {self.on_calib_error=}; '
+                                     f'valid options are {EMQ.ON_CALIB_ERROR_VALUES}')
+            return calib_posteriors
+        return uncalib_posteriors
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        if self.recalib is not None:
-            P, y = classif_predictions.Xy
-            if self.recalib == 'nbvs':
-                calibrator = NoBiasVectorScaling()
-            elif self.recalib == 'bcts':
-                calibrator = TempScaling(bias_positions='all')
-            elif self.recalib == 'ts':
-                calibrator = TempScaling()
-            elif self.recalib == 'vs':
-                calibrator = VectorScaling()
-            else:
-                raise ValueError('invalid param argument for recalibration method; available ones are '
-                                 '"nbvs", "bcts", "ts", and "vs".')
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of EMQ. This comes down to recalibrating the posterior probabilities
+        ir requested.
 
-            self.calibration_function = calibrator(P, np.eye(data.n_classes)[y], posterior_supplied=True)
+        :param classif_predictions: array-like with the raw (i.e., uncalibrated) posterior probabilities
+            returned by the classifier
+        :param labels: array-like with the true labels associated to each classifier prediction
+        """
+        P = classif_predictions
+        y = labels
 
-        if self.exact_train_prev:
-            self.train_prevalence = data.prevalence()
-        else:
-            train_posteriors = classif_predictions.X
-            if self.recalib is not None:
-                train_posteriors = self.calibration_function(train_posteriors)
-            self.train_prevalence = F.prevalence_from_probabilities(train_posteriors)

+        requires_predictions = (self.calib is not None) or (not self.exact_train_prev)
+        if P is None and requires_predictions:
+            # classifier predictions were not generated because val_split=None
+            raise ArgumentError(self.val_split, self.__class__.__name__ +
+                                ": Classifier predictions for the aggregative fit were not generated because "
+                                "val_split=None. This usually happens when you enable calibrations or heuristics "
+                                "during model selection but left val_split set to its default value (None). "
+                                "Please provide one of the following values for val_split: (i) an integer >1 "
+                                "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. "
+                                "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit "
+                                "validation data")
 
-
[docs]    def aggregate(self, classif_posteriors, epsilon=EPSILON):
+        if self.calib is not None:
+            calibrator = _get_abstention_calibrators().get(self.calib, None)
+
+            if calibrator is None:
+                raise ValueError(f'invalid value for {self.calib=}; valid ones are {EMQ.CALIB_OPTIONS}')
+
+            self._fit_calibration(calibrator, P, y)
+
+        if not self.exact_train_prev:
+            P = self._calibrate_if_requested(P)
+            self.train_prevalence = F.prevalence_from_probabilities(P)

+
+
+

+[docs]
+    def aggregate(self, classif_posteriors, epsilon=EPSILON):
+        classif_posteriors = self._calibrate_if_requested(classif_posteriors)
         priors, posteriors = self.EM(self.train_prevalence, classif_posteriors, epsilon)
         return priors

 
-
[docs]    def predict_proba(self, instances, epsilon=EPSILON):
+
+

+[docs]
+    def predict_proba(self, instances, epsilon=EPSILON):
         """
         Returns the posterior probabilities updated by the EM algorithm.
 
@@ -736,11 +1161,15 @@
         :return: np.ndarray of shape `(n_instances, n_classes)`
         """
         classif_posteriors = self.classify(instances)
+        classif_posteriors = self._calibrate_if_requested(classif_posteriors)
         priors, posteriors = self.EM(self.train_prevalence, classif_posteriors, epsilon)
         return posteriors

 
-
[docs]    @classmethod
-    def EM(cls, tr_prev, posterior_probabilities, epsilon=EPSILON):
+
+

+[docs]
+    @classmethod
+    def EM(cls, tr_prev, posterior_probabilities, epsilon=EPSILON):
         """
         Computes the `Expectation Maximization` routine.
 
@@ -754,6 +1183,11 @@
         """
         Px = posterior_probabilities
         Ptr = np.copy(tr_prev)
+
+        if np.prod(Ptr) == 0:  # some entry is 0; we should smooth the values to avoid 0 division
+            Ptr += epsilon
+            Ptr /= Ptr.sum()
+
         qs = np.copy(Ptr)  # qs (the running estimate) is initialized as the training prevalence
 
         s, converged = 0, False
@@ -775,10 +1209,14 @@
         if not converged:
             print('[warning] the method has reached the maximum number of iterations; it might have not converged')
 
-        return qs, ps

+        return qs, ps

+

 
 
-
[docs]class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `Hellinger Distance y <https://www.sciencedirect.com/science/article/pii/S0020025512004069>`_ (HDy).
     HDy is a probabilistic method for training binary quantifiers, that models quantification as the problem of
@@ -788,27 +1226,32 @@
     class-conditional distributions of the posterior probabilities returned for the positive and negative validation
     examples, respectively. The parameters of the mixture thus represent the estimates of the class prevalence values.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
-        Trains a HDy quantifier.
+        Trains the aggregation function of HDy.
 
-        :param data: the training set
-        :param fit_classifier: set to False to bypass the training (the learner is assumed to be already fit)
-        :param val_split: either a float in (0,1) indicating the proportion of training instances to use for
-         validation (e.g., 0.3 for using 30% of the training set as validation data), or a
-         :class:`quapy.data.base.LabelledCollection` indicating the validation set itself
-        :return: self
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
         """
-        P, y = classif_predictions.Xy
+        P, y = classif_predictions, labels
         Px = P[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
@@ -816,16 +1259,17 @@
         # pre-compute the histogram for positive and negative examples
         self.bins = np.linspace(10, 110, 11, dtype=int)  # [10, 20, 30, ..., 100, 110]
 
-        def hist(P, bins):
+        def hist(P, bins):
             h = np.histogram(P, bins=bins, range=(0, 1), density=True)[0]
             return h / h.sum()
 
         self.Pxy1_density = {bins: hist(self.Pxy1, bins) for bins in self.bins}
-        self.Pxy0_density = {bins: hist(self.Pxy0, bins) for bins in self.bins}
+        self.Pxy0_density = {bins: hist(self.Pxy0, bins) for bins in self.bins}

 
-        return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+

+[docs]
+    def aggregate(self, classif_posteriors):
         # "In this work, the number of bins b used in HDx and HDy was chosen from 10 to 110 in steps of 10,
         # and the final estimated a priori probability was taken as the median of these 11 estimates."
         # (González-Castro, et al., 2013).
@@ -846,7 +1290,7 @@
             # at small steps (modern implementations resort to an optimization procedure,
             # see class DistributionMatching)
             prev_selected, min_dist = None, None
-            for prev in F.prevalence_linspace(n_prevalences=101, repeats=1, smooth_limits_epsilon=0.0):
+            for prev in F.prevalence_linspace(grid_points=101, repeats=1, smooth_limits_epsilon=0.0):
                 Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density
                 hdy = F.HellingerDistance(Px_train, Px_test)
                 if prev_selected is None or hdy < min_dist:
@@ -854,35 +1298,51 @@
             prev_estimations.append(prev_selected)
 
         class1_prev = np.median(prev_estimations)
-        return F.as_binary_prevalence(class1_prev)

+        return F.as_binary_prevalence(class1_prev)

+

 
 
-
[docs]class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `DyS framework <https://ojs.aaai.org/index.php/AAAI/article/view/4376>`_ (DyS).
     DyS is a generalization of HDy method, using a Ternary Search in order to find the prevalence that
     minimizes the distance between distributions.
     Details for the ternary search have been got from <https://dl.acm.org/doi/pdf/10.1145/3219819.3220059>
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     :param n_bins: an int with the number of bins to use to compute the histograms.
+
     :param divergence: a str indicating the name of divergence (currently supported ones are "HD" or "topsoe"), or a
         callable function computes the divergence between two distributions (two equally sized arrays).
+
     :param tol: a float with the tolerance for the ternary search algorithm.
+
     :param n_jobs: number of parallel workers.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_bins=8, divergence: Union[str, Callable]= 'HD', tol=1e-05, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_bins=8,
+                 divergence: Union[str, Callable] = 'HD', tol=1e-05, n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.tol = tol
         self.divergence = divergence
         self.n_bins = n_bins
         self.n_jobs = n_jobs
 
-    def _ternary_search(self, f, left, right, tol):
+    def _ternary_search(self, f, left, right, tol):
         """
         Find maximum of unimodal function f() within [left, right]
         """
@@ -898,8 +1358,16 @@
         # Left and right are the current bounds; the maximum is between them
         return (left + right) / 2
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        Px, y = classif_predictions.Xy
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of DyS.
+
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
+        """
+        Px, y = classif_predictions, labels
         Px = Px[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
@@ -907,77 +1375,113 @@
         self.Pxy0_density = np.histogram(self.Pxy0, bins=self.n_bins, range=(0, 1), density=True)[0]
         return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
 
         Px_test = np.histogram(Px, bins=self.n_bins, range=(0, 1), density=True)[0]
         divergence = get_divergence(self.divergence)
 
-        def distribution_distance(prev):
+        def distribution_distance(prev):
             Px_train = prev * self.Pxy1_density + (1 - prev) * self.Pxy0_density
             return divergence(Px_train, Px_test)
-            
+
         class1_prev = self._ternary_search(f=distribution_distance, left=0, right=1, tol=self.tol)
-        return F.as_binary_prevalence(class1_prev)

+        return F.as_binary_prevalence(class1_prev)

+

 
 
-
[docs]class SMM(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class SMM(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `SMM method <https://ieeexplore.ieee.org/document/9260028>`_ (SMM).
     SMM is a simplification of matching distribution methods where the representation of the examples
     is created using the mean instead of a histogram (conceptually equivalent to PACC).
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        self.classifier = classifier
-        self.val_split = val_split
-      
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        Px, y = classif_predictions.Xy
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of SMM.
+
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
+        """
+        Px, y = classif_predictions, labels
         Px = Px[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
-        self.Pxy1_mean = np.mean(self.Pxy1)  # equiv. TPR 
+        self.Pxy1_mean = np.mean(self.Pxy1)  # equiv. TPR
         self.Pxy0_mean = np.mean(self.Pxy0)  # equiv. FPR
         return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
         Px_mean = np.mean(Px)
-     
-        class1_prev = (Px_mean - self.Pxy0_mean)/(self.Pxy1_mean - self.Pxy0_mean)
-        return F.as_binary_prevalence(class1_prev, clip_if_necessary=True)

+
+        class1_prev = (Px_mean - self.Pxy0_mean) / (self.Pxy1_mean - self.Pxy0_mean)
+        return F.as_binary_prevalence(class1_prev, clip_if_necessary=True)

+

 
 
-
[docs]class DMy(AggregativeSoftQuantifier):
+
+

+[docs]
+class DMy(AggregativeSoftQuantifier):
     """
     Generic Distribution Matching quantifier for binary or multiclass quantification based on the space of posterior
     probabilities. This implementation takes the number of bins, the divergence, and the possibility to work on CDF
     as hyperparameters.
 
-    :param classifier: a `sklearn`'s Estimator that generates a probabilistic classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set to model the
-        validation distribution.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the validation distribution should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     :param nbins: number of bins used to discretize the distributions (default 8)
+
     :param divergence: a string representing a divergence measure (currently, "HD" and "topsoe" are implemented)
         or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger
         Distance)
+
     :param cdf: whether to use CDF instead of PDF (default False)
+
     :param n_jobs: number of parallel workers (default None)
     """
 
-    def __init__(self, classifier, val_split=5, nbins=8, divergence: Union[str, Callable]='HD',
-                 cdf=False, search='optim_minimize', n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, nbins=8,
+                 divergence: Union[str, Callable] = 'HD', cdf=False, search='optim_minimize', n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.nbins = nbins
         self.divergence = divergence
         self.cdf = cdf
@@ -992,7 +1496,7 @@
     #     hdy = AggregativeMedianEstimator(hdy, param_grid={'nbins': np.linspace(10, 110, 11).astype(int)}, n_jobs=n_jobs)
     #     return hdy
 
-    def _get_distributions(self, posteriors):
+    def _get_distributions(self, posteriors):
         histograms = []
         post_dims = posteriors.shape[1]
         if post_dims == 2:
@@ -1003,38 +1507,40 @@
             histograms.append(hist)
 
         counts = np.vstack(histograms)
-        distributions = counts/counts.sum(axis=1)[:,np.newaxis]
+        distributions = counts / counts.sum(axis=1)[:, np.newaxis]
         if self.cdf:
             distributions = np.cumsum(distributions, axis=1)
         return distributions
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
-        Trains the classifier (if requested) and generates the validation distributions out of the training data.
+        Trains the aggregation function of a distribution matching method. This comes down to generating the
+        validation distributions out of the training data.
         The validation distributions have shape `(n, ch, nbins)`, with `n` the number of classes, `ch` the number of
         channels, and `nbins` the number of bins. In particular, let `V` be the validation distributions; then `di=V[i]`
         are the distributions obtained from training data labelled with class `i`; while `dij = di[j]` is the discrete
         distribution of posterior probabilities `P(Y=j|X=x)` for training data labelled with class `i`, and `dij[k]`
         is the fraction of instances with a value in the `k`-th bin.
 
-        :param data: the training set
-        :param fit_classifier: set to False to bypass the training (the learner is assumed to be already fit)
-        :param val_split: either a float in (0,1) indicating the proportion of training instances to use for
-         validation (e.g., 0.3 for using 30% of the training set as validation data), or a LabelledCollection
-         indicating the validation set itself, or an int indicating the number k of folds to be used in kFCV
-         to estimate the parameters
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
         """
-        posteriors, true_labels = classif_predictions.Xy
+        posteriors, true_labels = classif_predictions, labels
         n_classes = len(self.classifier.classes_)
 
         self.validation_distribution = qp.util.parallel(
             func=self._get_distributions,
-            args=[posteriors[true_labels==cat] for cat in range(n_classes)],
+            args=[posteriors[true_labels == cat] for cat in range(n_classes)],
             n_jobs=self.n_jobs,
             backend='threading'
         )

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         """
         Searches for the mixture model parameter (the sought prevalence values) that yields a validation distribution
         (the mixture) that best matches the test distribution, in terms of the divergence measure of choice.
@@ -1048,17 +1554,21 @@
         test_distribution = self._get_distributions(posteriors)
         divergence = get_divergence(self.divergence)
         n_classes, n_channels, nbins = self.validation_distribution.shape
-        def loss(prev):
+
+        def loss(prev):
             prev = np.expand_dims(prev, axis=0)
-            mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes,-1)).reshape(n_channels, -1)
+            mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes, -1)).reshape(n_channels, -1)
             divs = [divergence(test_distribution[ch], mixture_distribution[ch]) for ch in range(n_channels)]
             return np.mean(divs)
 
-        return F.argmin_prevalence(loss, n_classes, method=self.search)

+        return F.argmin_prevalence(loss, n_classes, method=self.search)

+

 
 
 
-
[docs]def newELM(svmperf_base=None, loss='01', C=1):
+

+[docs]
+def newELM(svmperf_base=None, loss='01', C=1):
     """
     Explicit Loss Minimization (ELM) quantifiers.
     Quantifiers based on ELM represent a family of methods based on structured output learning;
@@ -1085,7 +1595,10 @@
     return CC(SVMperf(svmperf_base, loss=loss, C=C))

 
 
-
[docs]def newSVMQ(svmperf_base=None, C=1):
+
+

+[docs]
+def newSVMQ(svmperf_base=None, C=1):
     """
     SVM(Q) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the `Q` loss combining a
     classification-oriented loss and a quantification-oriented loss, as proposed by
@@ -1110,7 +1623,9 @@
     """
     return newELM(svmperf_base, loss='q', C=C)

 
-def newSVMKLD(svmperf_base=None, C=1):
+
+
+def newSVMKLD(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
     as proposed by `Esuli et al. 2015 <https://dl.acm.org/doi/abs/10.1145/2700406>`_.
@@ -1135,7 +1650,9 @@
     return newELM(svmperf_base, loss='kld', C=C)
 
 
-
[docs]def newSVMKLD(svmperf_base=None, C=1):
+

+[docs]
+def newSVMKLD(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
     normalized via the logistic function, as proposed by
@@ -1160,7 +1677,11 @@
     """
     return newELM(svmperf_base, loss='nkld', C=C)

 
-
[docs]def newSVMAE(svmperf_base=None, C=1):
+
+
+

+[docs]
+def newSVMAE(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Absolute Error as first used by
     `Moreo and Sebastiani, 2021 <https://arxiv.org/abs/2011.02552>`_.
@@ -1184,7 +1705,11 @@
     """
     return newELM(svmperf_base, loss='mae', C=C)

 
-
[docs]def newSVMRAE(svmperf_base=None, C=1):
+
+
+

+[docs]
+def newSVMRAE(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Relative Absolute Error as first
     used by `Moreo and Sebastiani, 2021 <https://arxiv.org/abs/2011.02552>`_.
@@ -1209,7 +1734,10 @@
     return newELM(svmperf_base, loss='mrae', C=C)

 
 
-
[docs]class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
+
+

+[docs]
+class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
     """
     Allows any binary quantifier to perform quantification on single-label datasets.
     The method maintains one binary quantifier for each class, and then l1-normalizes the outputs so that the
@@ -1218,56 +1746,78 @@
     `Gao and Sebastiani, 2016 <https://link.springer.com/content/pdf/10.1007/s13278-016-0327-z.pdf>`_.
 
     :param binary_quantifier: a quantifier (binary) that will be employed to work on multiclass model in a
-        one-vs-all manner
+        one-vs-all manner (default PACC(LogitsticRegression()))
     :param n_jobs: number of parallel workers
     :param parallel_backend: the parallel backend for joblib (default "loky"); this is helpful for some quantifiers
         (e.g., ELM-based ones) that cannot be run with multiprocessing, since the temp dir they create during fit will
         is removed and no longer available at predict time.
     """
 
-    def __init__(self, binary_quantifier, n_jobs=None, parallel_backend='multiprocessing'):
+    def __init__(self, binary_quantifier=None, n_jobs=None, parallel_backend='multiprocessing'):
+        if binary_quantifier is None:
+            binary_quantifier = PACC()
         assert isinstance(binary_quantifier, BaseQuantifier), \
-            f'{self.binary_quantifier} does not seem to be a Quantifier'
+            f'{binary_quantifier} does not seem to be a Quantifier'
         assert isinstance(binary_quantifier, AggregativeQuantifier), \
-            f'{self.binary_quantifier} does not seem to be of type Aggregative'
+            f'{binary_quantifier} does not seem to be of type Aggregative'
         self.binary_quantifier = binary_quantifier
         self.n_jobs = qp._get_njobs(n_jobs)
         self.parallel_backend = parallel_backend
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
         If the base quantifier is not probabilistic, returns a matrix of shape `(n,m,)` with `n` the number of
         instances and `m` the number of classes. The entry `(i,j)` is a binary value indicating whether instance
-        `i `belongs to class `j`. The binary classifications are independent of each other, meaning that an instance
+        `i` belongs to class `j`. The binary classifications are independent of each other, meaning that an instance
         can end up be attributed to 0, 1, or more classes.
         If the base quantifier is probabilistic, returns a matrix of shape `(n,m,2)` with `n` the number of instances
         and `m` the number of classes. The entry `(i,j,1)` (resp. `(i,j,0)`) is a value in [0,1] indicating the
         posterior probability that instance `i` belongs (resp. does not belong) to class `j`. The posterior
         probabilities are independent of each other, meaning that, in general, they do not sum up to one.
 
-        :param instances: array-like
+        :param X: array-like
         :return: `np.ndarray`
         """
 
-        classif_predictions = self._parallel(self._delayed_binary_classification, instances)
+        classif_predictions = self._parallel(self._delayed_binary_classification, X)
         if isinstance(self.binary_quantifier, AggregativeSoftQuantifier):
             return np.swapaxes(classif_predictions, 0, 1)
         else:
             return classif_predictions.T

 
-
[docs]    def aggregate(self, classif_predictions):
+
+

+[docs]
+    def aggregate(self, classif_predictions):
         prevalences = self._parallel(self._delayed_binary_aggregate, classif_predictions)
         return F.normalize_prevalence(prevalences)

 
-    def _delayed_binary_classification(self, c, X):
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self._parallel(self._delayed_binary_aggregate_fit(c, classif_predictions, labels))
+        return self

+
+
+    def _delayed_binary_classification(self, c, X):
         return self.dict_binary_quantifiers[c].classify(X)
 
-    def _delayed_binary_aggregate(self, c, classif_predictions):
+    def _delayed_binary_aggregate(self, c, classif_predictions):
         # the estimation for the positive class prevalence
-        return self.dict_binary_quantifiers[c].aggregate(classif_predictions[:, c])[1]

+        return self.dict_binary_quantifiers[c].aggregate(classif_predictions[:, c])[1]
+
+    def _delayed_binary_aggregate_fit(self, c, classif_predictions, labels):
+        # trains the aggregation function of the cth quantifier
+        return self.dict_binary_quantifiers[c].aggregate_fit(classif_predictions[:, c], labels)

 
 
-
[docs]class AggregativeMedianEstimator(BinaryQuantifier):
+
+

+[docs]
+class AggregativeMedianEstimator(BinaryQuantifier):
     """
     This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
     estimation returned by differently (hyper)parameterized base quantifiers.
@@ -1277,51 +1827,57 @@
     :param base_quantifier: the base, binary quantifier
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param param_grid: the grid or parameters towards which the median will be computed
-    :param n_jobs: number of parllel workes
+    :param n_jobs: number of parallel workers
     """
-    def __init__(self, base_quantifier: AggregativeQuantifier, param_grid: dict, random_state=None, n_jobs=None):
+
+    def __init__(self, base_quantifier: AggregativeQuantifier, param_grid: dict, random_state=None, n_jobs=None):
         self.base_quantifier = base_quantifier
         self.param_grid = param_grid
         self.random_state = random_state
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         return self.base_quantifier.get_params(deep)

 
-
[docs]    def set_params(self, **params):
+
+

+[docs]
+    def set_params(self, **params):
         self.base_quantifier.set_params(**params)

 
-    def _delayed_fit(self, args):
+
+    def _delayed_fit(self, args):
         with qp.util.temp_seed(self.random_state):
-            params, training = args
+            params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**params)
-            model.fit(training)
+            model.fit(X, y)
             return model
 
-    def _delayed_fit_classifier(self, args):
+    def _delayed_fit_classifier(self, args):
         with qp.util.temp_seed(self.random_state):
-            print('enter job')
-            cls_params, training, kwargs = args
+            cls_params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(training, **kwargs)
-            print('exit job')
-            return (model, predictions)
+            predictions, labels = model.classifier_fit_predict(X, y)
+            return (model, predictions, labels)
 
-    def _delayed_fit_aggregation(self, args):
+    def _delayed_fit_aggregation(self, args):
         with qp.util.temp_seed(self.random_state):
-            ((model, predictions), q_params), training = args
+            ((model, predictions, y), q_params) = args
             model = deepcopy(model)
             model.set_params(**q_params)
-            model.aggregation_fit(predictions, training)
+            model.aggregation_fit(predictions, y)
             return model
 
+

+[docs]
+    def fit(self, X, y):
+        import itertools
 
-
[docs]    def fit(self, training: LabelledCollection, **kwargs):
-        import itertools
-
-        self._check_binary(training, self.__class__.__name__)
+        self._check_binary(y, self.__class__.__name__)
 
         if isinstance(self.base_quantifier, AggregativeQuantifier):
             cls_configs, q_configs = qp.model_selection.group_params(self.param_grid)
@@ -1329,22 +1885,21 @@
             if len(cls_configs) > 1:
                 models_preds = qp.util.parallel(
                     self._delayed_fit_classifier,
-                    ((params, training, kwargs) for params in cls_configs),
+                    ((params, X, y) for params in cls_configs),
                     seed=qp.environ.get('_R_SEED', None),
                     n_jobs=self.n_jobs,
                     asarray=False,
                     backend='threading'
                 )
             else:
-                print('only 1')
                 model = self.base_quantifier
                 model.set_params(**cls_configs[0])
-                predictions = model.classifier_fit_predict(training, **kwargs)
-                models_preds = [(model, predictions)]
+                predictions, labels = model.classifier_fit_predict(X, y)
+                models_preds = [(model, predictions, labels)]
 
             self.models = qp.util.parallel(
                 self._delayed_fit_aggregation,
-                ((setup, training) for setup in itertools.product(models_preds, q_configs)),
+                itertools.product(models_preds, q_configs),
                 seed=qp.environ.get('_R_SEED', None),
                 n_jobs=self.n_jobs,
                 backend='threading'
@@ -1353,18 +1908,21 @@
             configs = qp.model_selection.expand_grid(self.param_grid)
             self.models = qp.util.parallel(
                 self._delayed_fit,
-                ((params, training) for params in configs),
+                ((params, X, y) for params in configs),
                 seed=qp.environ.get('_R_SEED', None),
                 n_jobs=self.n_jobs,
                 backend='threading'
             )
         return self

 
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
 
-
[docs]    def quantify(self, instances):
+    def _delayed_predict(self, args):
+        model, instances = args
+        return model.predict(instances)
+
+

+[docs]
+    def predict(self, instances):
         prev_preds = qp.util.parallel(
             self._delayed_predict,
             ((model, instances) for model in self.models),
@@ -1372,39 +1930,40 @@
             n_jobs=self.n_jobs,
             backend='threading'
         )
-        return np.median(prev_preds, axis=0)

+        return np.median(prev_preds, axis=0)

+

 
 
-#---------------------------------------------------------------
+
+# ---------------------------------------------------------------
 # imports
-#---------------------------------------------------------------
+# ---------------------------------------------------------------
 
-from . import _threshold_optim
+from . import _threshold_optim
 
 T50 = _threshold_optim.T50
 MAX = _threshold_optim.MAX
-X   = _threshold_optim.X
-MS  = _threshold_optim.MS
+X = _threshold_optim.X
+MS = _threshold_optim.MS
 MS2 = _threshold_optim.MS2
 
-
-from . import _kdey
+from . import _kdey
 
 KDEyML = _kdey.KDEyML
 KDEyHD = _kdey.KDEyHD
 KDEyCS = _kdey.KDEyCS
 
-#---------------------------------------------------------------
+# ---------------------------------------------------------------
 # aliases
-#---------------------------------------------------------------
+# ---------------------------------------------------------------
 
 ClassifyAndCount = CC
 AdjustedClassifyAndCount = ACC
 ProbabilisticClassifyAndCount = PCC
 ProbabilisticAdjustedClassifyAndCount = PACC
 ExpectationMaximizationQuantifier = EMQ
-DistributionMatchingY = DMy
 SLD = EMQ
+DistributionMatchingY = DMy
 HellingerDistanceY = HDy
 MedianSweep = MS
 MedianSweep2 = MS2
diff --git a/docs/build/html/_modules/quapy/method/base.html b/docs/build/html/_modules/quapy/method/base.html
index 6288bd1..e649c5b 100644
--- a/docs/build/html/_modules/quapy/method/base.html
+++ b/docs/build/html/_modules/quapy/method/base.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.method.base — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.method.base — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,63 +74,94 @@
            

              
   
Source code for quapy.method.base
-from abc import ABCMeta, abstractmethod
-from copy import deepcopy
+from abc import ABCMeta, abstractmethod
+from copy import deepcopy
 
-from joblib import Parallel, delayed
-from sklearn.base import BaseEstimator
+from joblib import Parallel, delayed
+from sklearn.base import BaseEstimator
 
-import quapy as qp
-from quapy.data import LabelledCollection
-import numpy as np
+import quapy as qp
+from quapy.data import LabelledCollection
+import numpy as np
 
 
 # Base Quantifier abstract class
 # ------------------------------------
-
[docs]class BaseQuantifier(BaseEstimator):
+

+[docs]
+class BaseQuantifier(BaseEstimator):
     """
     Abstract Quantifier. A quantifier is defined as an object of a class that implements the method :meth:`fit` on
-    :class:`quapy.data.base.LabelledCollection`, the method :meth:`quantify`, and the :meth:`set_params` and
+    a pair X, y, the method :meth:`predict`, and the :meth:`set_params` and
     :meth:`get_params` for model selection (see :meth:`quapy.model_selection.GridSearchQ`)
     """
 
-
[docs]    @abstractmethod
-    def fit(self, data: LabelledCollection):
+

+[docs]
+    @abstractmethod
+    def fit(self, X, y):
         """
-        Trains a quantifier.
+        Generates a quantifier.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
+        :param X: array-like, the training instances
+        :param y: array-like, the labels
         :return: self
         """
         ...

 
-
[docs]    @abstractmethod
-    def quantify(self, instances):
+
+

+[docs]
+    @abstractmethod
+    def predict(self, X):
         """
         Generate class prevalence estimates for the sample's instances
 
-        :param instances: array-like
+        :param X: array-like, the test instances
         :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
         """
-        ...

+        ...

 
 
-
[docs]class BinaryQuantifier(BaseQuantifier):
+

+[docs]
+    def quantify(self, X):
+        """
+        Alias to :meth:`predict`, for old compatibility
+
+        :param X: array-like
+        :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
+        """
+        return self.predict(X)

+

+
+
+
+

+[docs]
+class BinaryQuantifier(BaseQuantifier):
     """
     Abstract class of binary quantifiers, i.e., quantifiers estimating class prevalence values for only two classes
     (typically, to be interpreted as one class and its complement).
     """
 
-    def _check_binary(self, data: LabelledCollection, quantifier_name):
-        assert data.binary, f'{quantifier_name} works only on problems of binary classification. ' \
+    def _check_binary(self, y, quantifier_name):
+        n_classes = len(set(y))
+        assert n_classes==2, f'{quantifier_name} works only on problems of binary classification. ' \
                             f'Use the class OneVsAll to enable {quantifier_name} work on single-label data.'

 
 
-
[docs]class OneVsAll:
+
+

+[docs]
+class OneVsAll:
     pass

 
 
-
[docs]def newOneVsAll(binary_quantifier, n_jobs=None):
+
+

+[docs]
+def newOneVsAll(binary_quantifier: BaseQuantifier, n_jobs=None):
     assert isinstance(binary_quantifier, BaseQuantifier), \
         f'{binary_quantifier} does not seem to be a Quantifier'
     if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier):
@@ -136,13 +170,16 @@
         return OneVsAllGeneric(binary_quantifier, n_jobs)

 
 
-
[docs]class OneVsAllGeneric(OneVsAll, BaseQuantifier):
+
+

+[docs]
+class OneVsAllGeneric(OneVsAll, BaseQuantifier):
     """
     Allows any binary quantifier to perform quantification on single-label datasets. The method maintains one binary
-    quantifier for each class, and then l1-normalizes the outputs so that the class prevelence values sum up to 1.
+    quantifier for each class, and then l1-normalizes the outputs so that the class prevalence values sum up to 1.
     """
 
-    def __init__(self, binary_quantifier, n_jobs=None):
+    def __init__(self, binary_quantifier: BaseQuantifier, n_jobs=None):
         assert isinstance(binary_quantifier, BaseQuantifier), \
             f'{binary_quantifier} does not seem to be a Quantifier'
         if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier):
@@ -151,35 +188,42 @@
         self.binary_quantifier = binary_quantifier
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True):
-        assert not data.binary, f'{self.__class__.__name__} expect non-binary data'
-        assert fit_classifier == True, 'fit_classifier must be True'
+

+[docs]
+    def fit(self, X, y):
+        self.classes = sorted(np.unique(y))
+        assert len(self.classes)!=2, f'{self.__class__.__name__} expect non-binary data'
 
-        self.dict_binary_quantifiers = {c: deepcopy(self.binary_quantifier) for c in data.classes_}
-        self._parallel(self._delayed_binary_fit, data)
+        self.dict_binary_quantifiers = {c: deepcopy(self.binary_quantifier) for c in self.classes}
+        self._parallel(self._delayed_binary_fit, X, y)
         return self

 
-    def _parallel(self, func, *args, **kwargs):
+
+    def _parallel(self, func, *args, **kwargs):
         return np.asarray(
             Parallel(n_jobs=self.n_jobs, backend='threading')(
-                delayed(func)(c, *args, **kwargs) for c in self.classes_
+                delayed(func)(c, *args, **kwargs) for c in self.classes
             )
         )
 
-
[docs]    def quantify(self, instances):
-        prevalences = self._parallel(self._delayed_binary_predict, instances)
+

+[docs]
+    def predict(self, X):
+        prevalences = self._parallel(self._delayed_binary_predict, X)
         return qp.functional.normalize_prevalence(prevalences)

 
-    @property
-    def classes_(self):
-        return sorted(self.dict_binary_quantifiers.keys())
 
-    def _delayed_binary_predict(self, c, X):
-        return self.dict_binary_quantifiers[c].quantify(X)[1]
+    # @property
+    # def classes_(self):
+    #     return sorted(self.dict_binary_quantifiers.keys())
+
+    def _delayed_binary_predict(self, c, X):
+        return self.dict_binary_quantifiers[c].predict(X)[1]
+
+    def _delayed_binary_fit(self, c, X, y):
+        bindata = LabelledCollection(X, y == c, classes=[False, True])
+        self.dict_binary_quantifiers[c].fit(*bindata.Xy)

 
-    def _delayed_binary_fit(self, c, data):
-        bindata = LabelledCollection(data.instances, data.labels == c, classes=[False, True])
-        self.dict_binary_quantifiers[c].fit(bindata)

 

 
            

diff --git a/docs/build/html/_modules/quapy/method/meta.html b/docs/build/html/_modules/quapy/method/meta.html
index ca38440..8ff0a2a 100644
--- a/docs/build/html/_modules/quapy/method/meta.html
+++ b/docs/build/html/_modules/quapy/method/meta.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.method.meta — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.method.meta — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,24 +74,24 @@
            

              
   
Source code for quapy.method.meta
-import itertools
-from copy import deepcopy
-from typing import Union
-import numpy as np
-from sklearn.linear_model import LogisticRegression
-from sklearn.metrics import f1_score, make_scorer, accuracy_score
-from sklearn.model_selection import GridSearchCV, cross_val_predict
-from tqdm import tqdm
+import itertools
+from copy import deepcopy
+from typing import Union, List
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import f1_score, make_scorer, accuracy_score
+from sklearn.model_selection import GridSearchCV, cross_val_predict
+from tqdm import tqdm
 
-import quapy as qp
-from quapy import functional as F
-from quapy.data import LabelledCollection
-from quapy.model_selection import GridSearchQ
-from quapy.method.base import BaseQuantifier, BinaryQuantifier
-from quapy.method.aggregative import CC, ACC, PACC, HDy, EMQ, AggregativeQuantifier
+import quapy as qp
+from quapy import functional as F
+from quapy.data import LabelledCollection
+from quapy.model_selection import GridSearchQ
+from quapy.method.base import BaseQuantifier, BinaryQuantifier
+from quapy.method.aggregative import CC, ACC, PACC, HDy, EMQ, AggregativeQuantifier, AggregativeSoftQuantifier
 
 try:
-    from . import _neural
+    from . import _neural
 except ModuleNotFoundError:
     _neural = None
 
@@ -99,7 +102,9 @@
     QuaNet = "QuaNet is not available due to missing torch package"
 
 
-
[docs]class MedianEstimator2(BinaryQuantifier):
+

+[docs]
+class MedianEstimator2(BinaryQuantifier):
     """
     This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
     estimation returned by differently (hyper)parameterized base quantifiers.
@@ -111,54 +116,69 @@
     :param param_grid: the grid or parameters towards which the median will be computed
     :param n_jobs: number of parllel workes
     """
-    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
+    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
         self.base_quantifier = base_quantifier
         self.param_grid = param_grid
         self.random_state = random_state
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         return self.base_quantifier.get_params(deep)

 
-
[docs]    def set_params(self, **params):
+
+

+[docs]
+    def set_params(self, **params):
         self.base_quantifier.set_params(**params)

 
-    def _delayed_fit(self, args):
+
+    def _delayed_fit(self, args):
         with qp.util.temp_seed(self.random_state):
-            params, training = args
+            params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**params)
-            model.fit(training)
+            model.fit(X, y)
             return model
 
-
[docs]    def fit(self, training: LabelledCollection):
-        self._check_binary(training, self.__class__.__name__)
+

+[docs]
+    def fit(self, X, y):
+        self._check_binary(y, self.__class__.__name__)
 
         configs = qp.model_selection.expand_grid(self.param_grid)
         self.models = qp.util.parallel(
             self._delayed_fit,
-            ((params, training) for params in configs),
+            ((params, X, y) for params in configs),
             seed=qp.environ.get('_R_SEED', None),
             n_jobs=self.n_jobs
         )
         return self

 
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
 
-
[docs]    def quantify(self, instances):
+    def _delayed_predict(self, args):
+        model, instances = args
+        return model.predict(instances)
+
+

+[docs]
+    def predict(self, X):
         prev_preds = qp.util.parallel(
             self._delayed_predict,
-            ((model, instances) for model in self.models),
+            ((model, X) for model in self.models),
             seed=qp.environ.get('_R_SEED', None),
             n_jobs=self.n_jobs
         )
         prev_preds = np.asarray(prev_preds)
-        return np.median(prev_preds, axis=0)

+        return np.median(prev_preds, axis=0)

+

 
 
-
[docs]class MedianEstimator(BinaryQuantifier):
+
+

+[docs]
+class MedianEstimator(BinaryQuantifier):
     """
     This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
     estimation returned by differently (hyper)parameterized base quantifiers.
@@ -168,100 +188,73 @@
     :param base_quantifier: the base, binary quantifier
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param param_grid: the grid or parameters towards which the median will be computed
-    :param n_jobs: number of parllel workes
+    :param n_jobs: number of parallel workers
     """
-    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
+    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
         self.base_quantifier = base_quantifier
         self.param_grid = param_grid
         self.random_state = random_state
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         return self.base_quantifier.get_params(deep)

 
-
[docs]    def set_params(self, **params):
+
+

+[docs]
+    def set_params(self, **params):
         self.base_quantifier.set_params(**params)

 
-    def _delayed_fit(self, args):
+
+    def _delayed_fit(self, args):
         with qp.util.temp_seed(self.random_state):
-            params, training = args
+            params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**params)
-            model.fit(training)
+            model.fit(X, y)
             return model
 
-    def _delayed_fit_classifier(self, args):
-        with qp.util.temp_seed(self.random_state):
-            cls_params, training = args
-            model = deepcopy(self.base_quantifier)
-            model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(training, predict_on=model.val_split)
-            return (model, predictions)
+

+[docs]
+    def fit(self, X, y):
+        self._check_binary(y, self.__class__.__name__)
 
-    def _delayed_fit_aggregation(self, args):
-        with qp.util.temp_seed(self.random_state):
-            ((model, predictions), q_params), training = args
-            model = deepcopy(model)
-            model.set_params(**q_params)
-            model.aggregation_fit(predictions, training)
-            return model
-
-
-
[docs]    def fit(self, training: LabelledCollection):
-        self._check_binary(training, self.__class__.__name__)
-
-        if isinstance(self.base_quantifier, AggregativeQuantifier):
-            cls_configs, q_configs = qp.model_selection.group_params(self.param_grid)
-
-            if len(cls_configs) > 1:
-                models_preds = qp.util.parallel(
-                    self._delayed_fit_classifier,
-                    ((params, training) for params in cls_configs),
-                    seed=qp.environ.get('_R_SEED', None),
-                    n_jobs=self.n_jobs,
-                    asarray=False
-                )
-            else:
-                model = self.base_quantifier
-                model.set_params(**cls_configs[0])
-                predictions = model.classifier_fit_predict(training, predict_on=model.val_split)
-                models_preds = [(model, predictions)]
-
-            self.models = qp.util.parallel(
-                self._delayed_fit_aggregation,
-                ((setup, training) for setup in itertools.product(models_preds, q_configs)),
-                seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                asarray=False
-            )
-        else:
-            configs = qp.model_selection.expand_grid(self.param_grid)
-            self.models = qp.util.parallel(
-                self._delayed_fit,
-                ((params, training) for params in configs),
-                seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                asarray=False
-            )
+        configs = qp.model_selection.expand_grid(self.param_grid)
+        self.models = qp.util.parallel(
+            self._delayed_fit,
+            ((params, X, y) for params in configs),
+            seed=qp.environ.get('_R_SEED', None),
+            n_jobs=self.n_jobs,
+            asarray=False
+        )
         return self

 
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
 
-
[docs]    def quantify(self, instances):
+    def _delayed_predict(self, args):
+        model, instances = args
+        return model.predict(instances)
+
+

+[docs]
+    def predict(self, X):
         prev_preds = qp.util.parallel(
             self._delayed_predict,
-            ((model, instances) for model in self.models),
+            ((model, X) for model in self.models),
             seed=qp.environ.get('_R_SEED', None),
             n_jobs=self.n_jobs,
             asarray=False
         )
         prev_preds = np.asarray(prev_preds)
-        return np.median(prev_preds, axis=0)

+        return np.median(prev_preds, axis=0)

+

 
 
-
[docs]class Ensemble(BaseQuantifier):
+
+

+[docs]
+class Ensemble(BaseQuantifier):
     VALID_POLICIES = {'ave', 'ptr', 'ds'} | qp.error.QUANTIFICATION_ERROR_NAMES
 
     """
@@ -301,7 +294,7 @@
     :param verbose: set to True (default is False) to get some information in standard output
     """
 
-    def __init__(self,
+    def __init__(self,
                  quantifier: BaseQuantifier,
                  size=50,
                  red_size=25,
@@ -326,17 +319,20 @@
         self.verbose = verbose
         self.max_sample_size = max_sample_size
 
-    def _sout(self, msg):
+    def _sout(self, msg):
         if self.verbose:
             print('[Ensemble]' + msg)
 
-
[docs]    def fit(self, data: qp.data.LabelledCollection, val_split: Union[qp.data.LabelledCollection, float] = None):
+

+[docs]
+    def fit(self, X, y):
+
+        data = LabelledCollection(X, y)
 
         if self.policy == 'ds' and not data.binary:
             raise ValueError(f'ds policy is only defined for binary quantification, but this dataset is not binary')
 
-        if val_split is None:
-            val_split = self.val_split
+        val_split = self.val_split
 
         # randomly chooses the prevalences for each member of the ensemble (preventing classes with less than
         # min_pos positive examples)
@@ -367,20 +363,26 @@
         self._sout('Fit [Done]')
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         predictions = np.asarray(
-            qp.util.parallel(_delayed_quantify, ((Qi, instances) for Qi in self.ensemble), n_jobs=self.n_jobs)
+            qp.util.parallel(_delayed_quantify, ((Qi, X) for Qi in self.ensemble), n_jobs=self.n_jobs)
         )
 
         if self.policy == 'ptr':
             predictions = self._ptr_policy(predictions)
         elif self.policy == 'ds':
-            predictions = self._ds_policy(predictions, instances)
+            predictions = self._ds_policy(predictions, X)
 
         predictions = np.mean(predictions, axis=0)
         return F.normalize_prevalence(predictions)

 
-
[docs]    def set_params(self, **parameters):
+
+

+[docs]
+    def set_params(self, **parameters):
         """
         This function should not be used within :class:`quapy.model_selection.GridSearchQ` (is here for compatibility
         with the abstract class).
@@ -396,7 +398,10 @@
                                   f'or Ensemble(Q(GridSearchCV(l))) with Q a quantifier class that has a classifier '
                                   f'l optimized for classification (not recommended).')

 
-
[docs]    def get_params(self, deep=True):
+
+

+[docs]
+    def get_params(self, deep=True):
         """
         This function should not be used within :class:`quapy.model_selection.GridSearchQ` (is here for compatibility
         with the abstract class).
@@ -410,13 +415,14 @@
 
         raise NotImplementedError()

 
-    def _accuracy_policy(self, error_name):
+
+    def _accuracy_policy(self, error_name):
         """
         Selects the red_size best performant quantifiers in a static way (i.e., dropping all non-selected instances).
         For each model in the ensemble, the performance is measured in terms of _error_name_ on the quantification of
         the samples used for training the rest of the models in the ensemble.
         """
-        from quapy.evaluation import evaluate_on_samples
+        from quapy.evaluation import evaluate_on_samples
         error = qp.error.from_name(error_name)
         tests = [m[3] for m in self.ensemble]
         scores = []
@@ -426,7 +432,7 @@
 
         self.ensemble = _select_k(self.ensemble, order, k=self.red_size)
 
-    def _ptr_policy(self, predictions):
+    def _ptr_policy(self, predictions):
         """
         Selects the predictions made by models that have been trained on samples with a prevalence that is most similar
         to a first approximation of the test prevalence as made by all models in the ensemble.
@@ -437,7 +443,7 @@
         order = np.argsort(ptr_differences)
         return _select_k(predictions, order, k=self.red_size)
 
-    def _ds_policy_get_posteriors(self, data: LabelledCollection):
+    def _ds_policy_get_posteriors(self, data: LabelledCollection):
         """
         In the original article, there are some aspects regarding this method that are not mentioned. The paper says
         that the distribution of posterior probabilities from training and test examples is compared by means of the
@@ -468,7 +474,7 @@
 
         return posteriors, posteriors_generator
 
-    def _ds_policy(self, predictions, test):
+    def _ds_policy(self, predictions, test):
         test_posteriors = self.post_proba_fn(test)
         test_distribution = get_probability_distribution(test_posteriors)
         tr_distributions = [m[2] for m in self.ensemble]
@@ -477,7 +483,7 @@
         return _select_k(predictions, order, k=self.red_size)
 
     @property
-    def aggregative(self):
+    def aggregative(self):
         """
         Indicates that the quantifier is not aggregative.
 
@@ -486,7 +492,7 @@
         return False
 
     @property
-    def probabilistic(self):
+    def probabilistic(self):
         """
         Indicates that the quantifier is not probabilistic.
 
@@ -495,7 +501,10 @@
         return False

 
 
-
[docs]def get_probability_distribution(posterior_probabilities, bins=8):
+
+

+[docs]
+def get_probability_distribution(posterior_probabilities, bins=8):
     """
     Gets a histogram out of the posterior probabilities (only for the binary case).
 
@@ -509,11 +518,12 @@
     return distribution

 
 
-def _select_k(elements, order, k):
+
+def _select_k(elements, order, k):
     return [elements[idx] for idx in order[:k]]
 
 
-def _delayed_new_instance(args):
+def _delayed_new_instance(args):
     base_quantifier, data, val_split, prev, posteriors, keep_samples, verbose, sample_size = args
     if verbose:
         print(f'\tfit-start for prev {F.strprev(prev)}, sample_size={sample_size}')
@@ -528,25 +538,25 @@
     sample = data.sampling_from_index(sample_index)
 
     if val_split is not None:
-        model.fit(sample, val_split=val_split)
+        model.fit(*sample.Xy, val_split=val_split)
     else:
-        model.fit(sample)
+        model.fit(*sample.Xy)
 
     tr_prevalence = sample.prevalence()
     tr_distribution = get_probability_distribution(posteriors[sample_index]) if (posteriors is not None) else None
 
     if verbose:
-        print(f'\t\--fit-ended for prev {F.strprev(prev)}')
+        print(f'\t--fit-ended for prev {F.strprev(prev)}')
 
     return (model, tr_prevalence, tr_distribution, sample if keep_samples else None)
 
 
-def _delayed_quantify(args):
+def _delayed_quantify(args):
     quantifier, instances = args
-    return quantifier[0].quantify(instances)
+    return quantifier[0].predict(instances)
 
 
-def _draw_simplex(ndim, min_val, max_trials=100):
+def _draw_simplex(ndim, min_val, max_trials=100):
     """
     Returns a uniform sampling from the ndim-dimensional simplex but guarantees that all dimensions
     are >= min_class_prev (for min_val>0, this makes the sampling not truly uniform)
@@ -571,7 +581,7 @@
                              f'>= {min_val} is unlikely (it failed after {max_trials} trials)')
 
 
-def _instantiate_ensemble(classifier, base_quantifier_class, param_grid, optim, param_model_sel, **kwargs):
+def _instantiate_ensemble(classifier, base_quantifier_class, param_grid, optim, param_model_sel, **kwargs):
     if optim is None:
         base_quantifier = base_quantifier_class(classifier)
     elif optim in qp.error.CLASSIFICATION_ERROR:
@@ -590,7 +600,7 @@
     return Ensemble(base_quantifier, **kwargs)
 
 
-def _check_error(error):
+def _check_error(error):
     if error is None:
         return None
     if error in qp.error.QUANTIFICATION_ERROR or error in qp.error.CLASSIFICATION_ERROR:
@@ -602,7 +612,9 @@
                          f'the name of an error function in {qp.error.ERROR_NAMES}')
 
 
-
[docs]def ensembleFactory(classifier, base_quantifier_class, param_grid=None, optim=None, param_model_sel: dict = None,
+

+[docs]
+def ensembleFactory(classifier, base_quantifier_class, param_grid=None, optim=None, param_model_sel: dict = None,
                     **kwargs):
     """
     Ensemble factory. Provides a unified interface for instantiating ensembles that can be optimized (via model
@@ -653,7 +665,10 @@
     return _instantiate_ensemble(classifier, base_quantifier_class, param_grid, error, param_model_sel, **kwargs)

 
 
-
[docs]def ECC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def ECC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.CC` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -676,7 +691,10 @@
     return ensembleFactory(classifier, CC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.ACC` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -699,7 +717,10 @@
     return ensembleFactory(classifier, ACC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EPACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EPACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.PACC` quantifiers.
 
@@ -721,7 +742,10 @@
     return ensembleFactory(classifier, PACC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EHDy(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EHDy(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.HDy` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -744,7 +768,10 @@
     return ensembleFactory(classifier, HDy, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EEMQ(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EEMQ(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.EMQ` quantifiers.
 
@@ -764,6 +791,141 @@
     """
 
     return ensembleFactory(classifier, EMQ, param_grid, optim, param_mod_sel, **kwargs)

+
+
+
+

+[docs]
+def merge(prev_predictions, merge_fun):
+    prev_predictions = np.asarray(prev_predictions)
+    if merge_fun == 'median':
+        prevalences = np.median(prev_predictions, axis=0)
+        prevalences = F.normalize_prevalence(prevalences, method='l1')
+    elif merge_fun == 'mean':
+        prevalences = np.mean(prev_predictions, axis=0)
+    else:
+        raise NotImplementedError(f'merge function {merge_fun} not implemented!')
+    return prevalences

+
+
+
+

+[docs]
+class SCMQ(AggregativeSoftQuantifier):
+
+    MERGE_FUNCTIONS = ['median', 'mean']
+
+    def __init__(self, classifier, quantifiers: List[AggregativeSoftQuantifier], merge_fun='median', val_split=5):
+        self.classifier = classifier
+        self.quantifiers = [deepcopy(q) for q in quantifiers]
+        assert merge_fun in self.MERGE_FUNCTIONS, f'unknown {merge_fun=}, valid ones are {self.MERGE_FUNCTIONS}'
+        self.merge_fun = merge_fun
+        self.val_split = val_split
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        for quantifier in self.quantifiers:
+            quantifier.classifier = self.classifier
+            quantifier.aggregation_fit(classif_predictions, labels)
+        return self

+
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
+        prev_predictions = []
+        for quantifier_i in self.quantifiers:
+            prevalence_i = quantifier_i.aggregate(classif_predictions)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+

+[docs]
+class MCSQ(BaseQuantifier):
+    def __init__(self, classifiers, quantifier: AggregativeSoftQuantifier, merge_fun='median', val_split=5):
+        self.merge_fun = merge_fun
+        self.val_split = val_split
+        self.mcsqs = []
+        for classifier in classifiers:
+            quantifier = deepcopy(quantifier)
+            quantifier.classifier = classifier
+            self.mcsqs.append(quantifier)
+
+

+[docs]
+    def fit(self, data: LabelledCollection):
+        for q in self.mcsqs:
+            q.fit(data, val_split=self.val_split)
+        return self

+
+
+

+[docs]
+    def quantify(self, instances):
+        prev_predictions = []
+        for q in self.mcsqs:
+            prevalence_i = q.quantify(instances)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+

+[docs]
+class MCMQ(BaseQuantifier):
+    def __init__(self, classifiers, quantifiers: List[AggregativeSoftQuantifier], merge_fun='median', val_split=5):
+        self.merge_fun = merge_fun
+        self.scmqs = []
+        for classifier in classifiers:
+            self.scmqs.append(SCMQ(classifier, quantifiers, val_split=val_split))
+
+

+[docs]
+    def fit(self, data: LabelledCollection):
+        for q in self.scmqs:
+            q.fit(data)
+        return self

+
+
+

+[docs]
+    def quantify(self, instances):
+        prev_predictions = []
+        for q in self.scmqs:
+            prevalence_i = q.quantify(instances)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 

 
            

diff --git a/docs/build/html/_modules/quapy/method/non_aggregative.html b/docs/build/html/_modules/quapy/method/non_aggregative.html
index aeb5b96..e02061b 100644
--- a/docs/build/html/_modules/quapy/method/non_aggregative.html
+++ b/docs/build/html/_modules/quapy/method/non_aggregative.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,16 +74,25 @@
            

              
   
Source code for quapy.method.non_aggregative
-from typing import Union, Callable
-import numpy as np
+from itertools import product
+from tqdm import tqdm
+from typing import Union, Callable, Counter
+import numpy as np
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.utils import resample
+from sklearn.preprocessing import normalize
 
-from quapy.functional import get_divergence
-from quapy.data import LabelledCollection
-from quapy.method.base import BaseQuantifier, BinaryQuantifier
-import quapy.functional as F
+from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC
+from quapy.functional import get_divergence
+from quapy.method.base import BaseQuantifier, BinaryQuantifier
+import quapy.functional as F
+from scipy.optimize import lsq_linear
+from scipy import sparse
 
 
-
[docs]class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier):
+

+[docs]
+class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier):
     """
     The `Maximum Likelihood Prevalence Estimation` (MLPE) method is a lazy method that assumes there is no prior
     probability shift between training and test instances (put it other way, that the i.i.d. assumpion holds).
@@ -89,30 +101,41 @@
     any quantification method should beat.
     """
 
-    def __init__(self):
+    def __init__(self):
         self._classes_ = None
 
-
[docs]    def fit(self, data: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """
         Computes the training prevalence and stores it.
 
-        :param data: the training sample
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         :return: self
         """
-        self.estimated_prevalence = data.prevalence()
+        self._classes_ = F.classes_from_labels(labels=y)
+        self.estimated_prevalence = F.prevalence_from_labels(y, classes=self._classes_)
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """
         Ignores the input instances and returns, as the class prevalence estimantes, the training prevalence.
 
-        :param instances: array-like (ignored)
+        :param X: array-like (ignored)
         :return: the class prevalence seen during training
         """
-        return self.estimated_prevalence

+        return self.estimated_prevalence

+

 
 
-
[docs]class DMx(BaseQuantifier):
+
+

+[docs]
+class DMx(BaseQuantifier):
     """
     Generic Distribution Matching quantifier for binary or multiclass quantification based on the space of covariates.
     This implementation takes the number of bins, the divergence, and the possibility to work on CDF as hyperparameters.
@@ -125,15 +148,17 @@
     :param n_jobs: number of parallel workers (default None)
     """
 
-    def __init__(self, nbins=8, divergence: Union[str, Callable]='HD', cdf=False, search='optim_minimize', n_jobs=None):
+    def __init__(self, nbins=8, divergence: Union[str, Callable]='HD', cdf=False, search='optim_minimize', n_jobs=None):
         self.nbins = nbins
         self.divergence = divergence
         self.cdf = cdf
         self.search = search
         self.n_jobs = n_jobs
 
-
[docs]    @classmethod
-    def HDx(cls, n_jobs=None):
+

+[docs]
+    @classmethod
+    def HDx(cls, n_jobs=None):
         """
         `Hellinger Distance x <https://www.sciencedirect.com/science/article/pii/S0020025512004069>`_ (HDx).
         HDx is a method for training binary quantifiers, that models quantification as the problem of
@@ -149,14 +174,15 @@
         :return: an instance of this class setup to mimick the performance of the HDx as originally proposed by
             González-Castro, Alaiz-Rodríguez, Alegre (2013)
         """
-        from quapy.method.meta import MedianEstimator
+        from quapy.method.meta import MedianEstimator
 
         dmx = DMx(divergence='HD', cdf=False, search='linear_search')
         nbins = {'nbins': np.linspace(10, 110, 11, dtype=int)}
         hdx = MedianEstimator(base_quantifier=dmx, param_grid=nbins, n_jobs=n_jobs)
         return hdx

 
-    def __get_distributions(self, X):
+
+    def __get_distributions(self, X):
 
         histograms = []
         for feat_idx in range(self.nfeats):
@@ -172,7 +198,9 @@
 
         return distributions
 
-
[docs]    def fit(self, data: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """
         Generates the validation distributions out of the training data (covariates).
         The validation distributions have shape `(n, nfeats, nbins)`, with `n` the number of classes, `nfeats`
@@ -181,46 +209,222 @@
         training data labelled with class `i`; while `dij = di[j]` is the discrete distribution for feature j in
         training data labelled with class `i`, and `dij[k]` is the fraction of instances with a value in the `k`-th bin.
 
-        :param data: the training set
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         """
-        X, y = data.Xy
-
         self.nfeats = X.shape[1]
         self.feat_ranges = _get_features_range(X)
+        n_classes = len(np.unique(y))
 
         self.validation_distribution = np.asarray(
-            [self.__get_distributions(X[y==cat]) for cat in range(data.n_classes)]
+            [self.__get_distributions(X[y==cat]) for cat in range(n_classes)]
         )
 
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """
         Searches for the mixture model parameter (the sought prevalence values) that yields a validation distribution
         (the mixture) that best matches the test distribution, in terms of the divergence measure of choice.
         The matching is computed as the average dissimilarity (in terms of the dissimilarity measure of choice)
         between all feature-specific discrete distributions.
 
-        :param instances: instances in the sample
+        :param X: instances in the sample
         :return: a vector of class prevalence estimates
         """
 
-        assert instances.shape[1] == self.nfeats, f'wrong shape; expected {self.nfeats}, found {instances.shape[1]}'
+        assert X.shape[1] == self.nfeats, f'wrong shape; expected {self.nfeats}, found {X.shape[1]}'
 
-        test_distribution = self.__get_distributions(instances)
+        test_distribution = self.__get_distributions(X)
         divergence = get_divergence(self.divergence)
         n_classes, n_feats, nbins = self.validation_distribution.shape
-        def loss(prev):
+        def loss(prev):
             prev = np.expand_dims(prev, axis=0)
             mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes,-1)).reshape(n_feats, -1)
             divs = [divergence(test_distribution[feat], mixture_distribution[feat]) for feat in range(n_feats)]
             return np.mean(divs)
 
-        return F.argmin_prevalence(loss, n_classes, method=self.search)

+        return F.argmin_prevalence(loss, n_classes, method=self.search)

+

 
 
 
-def _get_features_range(X):
+
+
+

+[docs]
+class ReadMe(BaseQuantifier, WithConfidenceABC):
+    """
+    ReadMe is a non-aggregative quantification system proposed by
+    `Daniel Hopkins and Gary King, 2007. A method of automated nonparametric content analysis for
+    social science. American Journal of Political Science, 54(1):229–247.
+    <https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1540-5907.2009.00428.x>`_.
+    The idea is to estimate `Q(Y=i)` directly from:
+
+    :math:`Q(X)=\\sum_{i=1} Q(X|Y=i) Q(Y=i)`
+
+    via least-squares regression, i.e., without incurring the cost of computing posterior probabilities.
+    However, this poses a very difficult representation in which the vector `Q(X)` and the matrix `Q(X|Y=i)`
+    can be of very high dimensions. In order to render the problem tracktable, ReadMe performs bagging in
+    the feature space. ReadMe also combines bagging with bootstrap in order to derive confidence intervals
+    around point estimations.
+
+    We use the same default parameters as in the official
+    `R implementation <https://github.com/iqss-research/ReadMeV1/blob/master/R/prototype.R>`_.
+
+    :param prob_model: str ('naive', or 'full'), selects the modality in which the probabilities `Q(X)` and
+        `Q(X|Y)` are to be modelled. Options include "full", which corresponds to the original formulation of
+        ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which
+        `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where
+        `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes.
+        Of course, this approach is computationally prohibited for large `K`, so the computation is restricted to data
+        matrices with `K<=25` (although we recommend even smaller values of `K`). A much faster model is "naive", which
+        considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this
+        case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior).
+    :param bootstrap_trials: int, number of bootstrap trials (default 300)
+    :param bagging_trials: int, number of bagging trials (default 300)
+    :param bagging_range: int, number of features to keep for each bagging trial (default 15)
+    :param confidence_level: float, a value in (0,1) reflecting the desired confidence level (default 0.95)
+    :param region: str in 'intervals', 'ellipse', 'ellipse-clr'; indicates the preferred method for
+        defining the confidence region (see :class:`WithConfidenceABC`)
+    :param random_state: int or None, allows replicability (default None)
+    :param verbose: bool, whether to display information during the process (default False)
+    """
+
+    MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION = 25
+    PROBABILISTIC_MODELS = ["naive", "full"]
+
+    def __init__(self,
+                 prob_model="full",
+                 bootstrap_trials=300,
+                 bagging_trials=300,
+                 bagging_range=15,
+                 confidence_level=0.95,
+                 region='intervals',
+                 random_state=None,
+                 verbose=False):
+        assert prob_model in ReadMe.PROBABILISTIC_MODELS, \
+            f'unknown {prob_model=}, valid ones are {ReadMe.PROBABILISTIC_MODELS=}'
+        self.prob_model = prob_model
+        self.bootstrap_trials = bootstrap_trials
+        self.bagging_trials = bagging_trials
+        self.bagging_range = bagging_range
+        self.confidence_level = confidence_level
+        self.region = region
+        self.random_state = random_state
+        self.verbose = verbose
+
+

+[docs]
+    def fit(self, X, y):
+        self._check_matrix(X)
+
+        self.rng = np.random.default_rng(self.random_state)
+        self.classes_ = np.unique(y)
+
+
+        Xsize = X.shape[0]
+
+        # Bootstrap loop
+        self.Xboots, self.yboots = [], []
+        for _ in range(self.bootstrap_trials):
+            idx = self.rng.choice(Xsize, size=Xsize, replace=True)
+            self.Xboots.append(X[idx])
+            self.yboots.append(y[idx])
+
+        return self

+
+
+

+[docs]
+    def predict_conf(self, X, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC):
+        self._check_matrix(X)
+
+        n_features = X.shape[1]
+        boots_prevalences = []
+        for Xboots, yboots in tqdm(
+                zip(self.Xboots, self.yboots),
+                desc='bootstrap predictions', total=self.bootstrap_trials, disable=not self.verbose
+        ):
+            bagging_estimates = []
+            for _ in range(self.bagging_trials):
+                feat_idx = self.rng.choice(n_features, size=self.bagging_range, replace=False)
+                Xboots_bagging = Xboots[:, feat_idx]
+                X_boots_bagging = X[:, feat_idx]
+                bagging_prev = self._quantify_iteration(Xboots_bagging, yboots, X_boots_bagging)
+                bagging_estimates.append(bagging_prev)
+
+            boots_prevalences.append(np.mean(bagging_estimates, axis=0))
+
+        conf = WithConfidenceABC.construct_region(boots_prevalences, confidence_level, method=self.region)
+        prev_estim = conf.point_estimate()
+
+        return prev_estim, conf

+
+
+

+[docs]
+    def predict(self, X):
+        prev_estim, _ = self.predict_conf(X)
+        return prev_estim

+
+
+    def _quantify_iteration(self, Xtr, ytr, Xte):
+        """Single ReadMe estimate."""
+        PX_given_Y = np.asarray([self._compute_P(Xtr[ytr == c]) for i,c in enumerate(self.classes_)])
+        PX = self._compute_P(Xte)
+
+        res = lsq_linear(A=PX_given_Y.T, b=PX, bounds=(0, 1))
+        pY = np.maximum(res.x, 0)
+        return pY / pY.sum()
+
+    def _check_matrix(self, X):
+        """the "full" model requires estimating empirical distributions; due to the high computational cost,
+                this function is only made available for binary matrices"""
+        if self.prob_model == 'full' and not self._is_binary_matrix(X):
+            raise ValueError('the empirical distribution can only be computed efficiently on binary matrices')
+
+    def _is_binary_matrix(self, X):
+        data = X.data if sparse.issparse(X) else X
+        return np.all((data == 0) | (data == 1))
+
+    def _compute_P(self, X):
+        if self.prob_model == 'naive':
+            return self._multinomial_distribution(X)
+        elif self.prob_model == 'full':
+            return self._empirical_distribution(X)
+        else:
+            raise ValueError(f'unknown {self.prob_model}; valid ones are {ReadMe.PROBABILISTIC_MODELS=}')
+
+    def _empirical_distribution(self, X):
+
+        if X.shape[1] > self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION:
+            raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions '
+                             f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}')
+
+        # we convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5)
+        K = X.shape[1]
+        binary_powers = 1 << np.arange(K-1, -1, -1)  # (2^K, ..., 32, 16, 8, 4, 2, 1)
+        X_as_binary_numbers = X @ binary_powers
+
+        # count occurrences and compute probs
+        counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float)
+        probs = counts / counts.sum()
+        return probs
+
+    def _multinomial_distribution(self, X):
+        PX = np.asarray(X.sum(axis=0))
+        PX = normalize(PX, norm='l1', axis=1)
+        return PX.ravel()

+
+
+
+
+
+def _get_features_range(X):
     feat_ranges = []
     ncols = X.shape[1]
     for col_idx in range(ncols):
diff --git a/docs/build/html/_modules/quapy/model_selection.html b/docs/build/html/_modules/quapy/model_selection.html
index 172c8f3..461c068 100644
--- a/docs/build/html/_modules/quapy/model_selection.html
+++ b/docs/build/html/_modules/quapy/model_selection.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.model_selection — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.model_selection — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,52 +74,67 @@
            

              
   
Source code for quapy.model_selection
-import itertools
-import signal
-from copy import deepcopy
-from enum import Enum
-from typing import Union, Callable
-from functools import wraps
+import itertools
+import signal
+from copy import deepcopy
+from enum import Enum
+from typing import Union, Callable
+from functools import wraps
 
-import numpy as np
-from sklearn import clone
+import numpy as np
+from sklearn import clone
 
-import quapy as qp
-from quapy import evaluation
-from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol
-from quapy.data.base import LabelledCollection
-from quapy.method.aggregative import BaseQuantifier, AggregativeQuantifier
-from quapy.util import timeout
-from time import time
+import quapy as qp
+from quapy import evaluation
+from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol
+from quapy.data.base import LabelledCollection
+from quapy.method.aggregative import BaseQuantifier, AggregativeQuantifier
+from quapy.util import timeout
+from time import time
 
 
-
[docs]class Status(Enum):
+

+[docs]
+class Status(Enum):
     SUCCESS = 1
     TIMEOUT = 2
     INVALID = 3
     ERROR = 4

 
 
-
[docs]class ConfigStatus:
-    def __init__(self, params, status, msg=''):
+
+

+[docs]
+class ConfigStatus:
+
+    def __init__(self, params, status, msg=''):
         self.params = params
         self.status = status
         self.msg = msg
 
-    def __str__(self):
+    def __str__(self):
         return f':params:{self.params} :status:{self.status} ' + self.msg
 
-    def __repr__(self):
+    def __repr__(self):
         return str(self)
 
-
[docs]    def success(self):
+

+[docs]
+    def success(self):
         return self.status == Status.SUCCESS

 
-
[docs]    def failed(self):
-        return self.status != Status.SUCCESS

+
+

+[docs]
+    def failed(self):
+        return self.status != Status.SUCCESS

+

 
 
-
[docs]class GridSearchQ(BaseQuantifier):
+
+

+[docs]
+class GridSearchQ(BaseQuantifier):
     """Grid Search optimization targeting a quantification-oriented metric.
 
     Optimizes the hyperparameters of a quantification method, based on an evaluation method and on an evaluation
@@ -139,7 +157,7 @@
     :param verbose: set to True to get information through the stdout
     """
 
-    def __init__(self,
+    def __init__(self,
                  model: BaseQuantifier,
                  param_grid: dict,
                  protocol: AbstractProtocol,
@@ -158,14 +176,14 @@
         self.n_jobs = qp._get_njobs(n_jobs)
         self.raise_errors = raise_errors
         self.verbose = verbose
-        self.__check_error(error)
+        self.__check_error_measure(error)
         assert isinstance(protocol, AbstractProtocol), 'unknown protocol'
 
-    def _sout(self, msg):
+    def _sout(self, msg):
         if self.verbose:
             print(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}')
 
-    def __check_error(self, error):
+    def __check_error_measure(self, error):
         if error in qp.error.QUANTIFICATION_ERROR:
             self.error = error
         elif isinstance(error, str):
@@ -176,26 +194,27 @@
             raise ValueError(f'unexpected error type; must either be a callable function or a str representing\n'
                              f'the name of an error function in {qp.error.QUANTIFICATION_ERROR_NAMES}')
 
-    def _prepare_classifier(self, cls_params):
+    def _prepare_classifier(self, cls_params):
         model = deepcopy(self.model)
 
-        def job(cls_params):
+        def job(cls_params):
             model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(self._training)
+            predictions = model.classifier_fit_predict(self._training_X, self._training_y)
             return predictions
 
         predictions, status, took = self._error_handler(job, cls_params)
         self._sout(f'[classifier fit] hyperparams={cls_params} [took {took:.3f}s]')
         return model, predictions, status, took
 
-    def _prepare_aggregation(self, args):
+    def _prepare_aggregation(self, args):
         model, predictions, cls_took, cls_params, q_params = args
         model = deepcopy(model)
         params = {**cls_params, **q_params}
 
-        def job(q_params):
+        def job(q_params):
             model.set_params(**q_params)
-            model.aggregation_fit(predictions, self._training)
+            P, y = predictions
+            model.aggregation_fit(P, y)
             score = evaluation.evaluate(model, protocol=self.protocol, error_metric=self.error)
             return score
 
@@ -203,12 +222,12 @@
         self._print_status(params, score, status, aggr_took)
         return model, params, score, status, (cls_took+aggr_took)
 
-    def _prepare_nonaggr_model(self, params):
+    def _prepare_nonaggr_model(self, params):
         model = deepcopy(self.model)
 
-        def job(params):
+        def job(params):
             model.set_params(**params)
-            model.fit(self._training)
+            model.fit(self._training_X, self._training_y)
             score = evaluation.evaluate(model, protocol=self.protocol, error_metric=self.error)
             return score
 
@@ -216,7 +235,7 @@
         self._print_status(params, score, status, took)
         return model, params, score, status, took
 
-    def _break_down_fit(self):
+    def _break_down_fit(self):
         """
         Decides whether to break down the fit phase in two (classifier-fit followed by aggregation-fit).
         In order to do so, some conditions should be met: a) the quantifier is of type aggregative,
@@ -231,17 +250,19 @@
             return False
         return True
 
-    def _compute_scores_aggregative(self, training):
+    def _compute_scores_aggregative(self, X, y):
         # break down the set of hyperparameters into two: classifier-specific, quantifier-specific
         cls_configs, q_configs = group_params(self.param_grid)
 
         # train all classifiers and get the predictions
-        self._training = training
+        self._training_X = X
+        self._training_y = y
         cls_outs = qp.util.parallel(
             self._prepare_classifier,
             cls_configs,
             seed=qp.environ.get('_R_SEED', None),
-            n_jobs=self.n_jobs
+            n_jobs=self.n_jobs,
+            asarray=False
         )
 
         # filter out classifier configurations that yielded any error
@@ -266,9 +287,10 @@
 
         return aggr_outs
 
-    def _compute_scores_nonaggregative(self, training):
+    def _compute_scores_nonaggregative(self, X, y):
         configs = expand_grid(self.param_grid)
-        self._training = training
+        self._training_X = X
+        self._training_y = y
         scores = qp.util.parallel(
             self._prepare_nonaggr_model,
             configs,
@@ -277,17 +299,20 @@
         )
         return scores
 
-    def _print_status(self, params, score, status, took):
+    def _print_status(self, params, score, status, took):
         if status.success():
             self._sout(f'hyperparams=[{params}]\t got {self.error.__name__} = {score:.5f} [took {took:.3f}s]')
         else:
             self._sout(f'error={status}')
 
-
[docs]    def fit(self, training: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """ Learning routine. Fits methods with all combinations of hyperparameters and selects the one minimizing
             the error metric.
 
-        :param training: the training set on which to optimize the hyperparameters
+        :param X: array-like, training covariates
+        :param y: array-like, labels of training data
         :return: self
         """
 
@@ -303,9 +328,9 @@
 
         self._sout(f'starting model selection with n_jobs={self.n_jobs}')
         if self._break_down_fit():
-            results = self._compute_scores_aggregative(training)
+            results = self._compute_scores_aggregative(X, y)
         else:
-            results = self._compute_scores_nonaggregative(training)
+            results = self._compute_scores_nonaggregative(X, y)
 
         self.param_scores_ = {}
         self.best_score_ = None
@@ -320,13 +345,13 @@
                 self.param_scores_[str(params)] = status.status
                 self.error_collector.append(status)
 
-        tend = time()-tinit
+        self.fit_time_ = time()-tinit
 
         if self.best_score_ is None:
             raise ValueError('no combination of hyperparameters seemed to work')
 
         self._sout(f'optimization finished: best params {self.best_params_} (score={self.best_score_:.5f}) '
-                   f'[took {tend:.4f}s]')
+                   f'[took {self.fit_time_:.4f}s]')
 
         no_errors = len(self.error_collector)
         if no_errors>0:
@@ -338,7 +363,10 @@
             if isinstance(self.protocol, OnLabelledCollectionProtocol):
                 tinit = time()
                 self._sout(f'refitting on the whole development set')
-                self.best_model_.fit(training + self.protocol.get_labelled_collection())
+                validation_collection = self.protocol.get_labelled_collection()
+                training_collection = LabelledCollection(X, y, classes=validation_collection.classes)
+                devel_collection = training_collection + validation_collection
+                self.best_model_.fit(*devel_collection.Xy)
                 tend = time() - tinit
                 self.refit_time_ = tend
             else:
@@ -347,24 +375,33 @@
 
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """Estimate class prevalence values using the best model found after calling the :meth:`fit` method.
 
-        :param instances: sample contanining the instances
+        :param X: sample contanining the instances
         :return: a ndarray of shape `(n_classes)` with class prevalence estimates as according to the best model found
             by the model selection process.
         """
         assert hasattr(self, 'best_model_'), 'quantify called before fit'
-        return self.best_model().quantify(instances)

+        return self.best_model().predict(X)

 
-
[docs]    def set_params(self, **parameters):
+
+

+[docs]
+    def set_params(self, **parameters):
         """Sets the hyper-parameters to explore.
 
         :param parameters: a dictionary with keys the parameter names and values the list of values to explore
         """
         self.param_grid = parameters

 
-
[docs]    def get_params(self, deep=True):
+
+

+[docs]
+    def get_params(self, deep=True):
         """Returns the dictionary of hyper-parameters to explore (`param_grid`)
 
         :param deep: Unused
@@ -372,7 +409,10 @@
         """
         return self.param_grid

 
-
[docs]    def best_model(self):
+
+

+[docs]
+    def best_model(self):
         """
         Returns the best model found after calling the :meth:`fit` method, i.e., the one trained on the combination
         of hyper-parameters that minimized the error function.
@@ -383,7 +423,8 @@
             return self.best_model_
         raise ValueError('best_model called before fit')

 
-    def _error_handler(self, func, params):
+
+    def _error_handler(self, func, params):
         """
         Endorses one job with two returned values: the status, and the time of execution
 
@@ -396,11 +437,11 @@
 
         output = None
 
-        def _handle(status, exception):
+        def _handle(status, exception):
             if self.raise_errors:
                 raise exception
             else:
-                return ConfigStatus(params, status)
+                return ConfigStatus(params, status, msg=str(exception))
 
         try:
             with timeout(self.timeout):
@@ -421,7 +462,10 @@
         return output, status, took

 
 
-
[docs]def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfolds=3, random_state=0):
+
+

+[docs]
+def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfolds=3, random_state=0):
     """
     Akin to `scikit-learn's cross_val_predict <https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_val_predict.html>`_
     but for quantification.
@@ -436,15 +480,18 @@
     total_prev = np.zeros(shape=data.n_classes)
 
     for train, test in data.kFCV(nfolds=nfolds, random_state=random_state):
-        quantifier.fit(train)
-        fold_prev = quantifier.quantify(test.X)
+        quantifier.fit(*train.Xy)
+        fold_prev = quantifier.predict(test.X)
         rel_size = 1. * len(test) / len(data)
         total_prev += fold_prev*rel_size
 
     return total_prev

 
 
-
[docs]def expand_grid(param_grid: dict):
+
+

+[docs]
+def expand_grid(param_grid: dict):
     """
     Expands a param_grid dictionary as a list of configurations.
     Example:
@@ -463,7 +510,10 @@
     return configs

 
 
-
[docs]def group_params(param_grid: dict):
+
+

+[docs]
+def group_params(param_grid: dict):
     """
     Partitions a param_grid dictionary as two lists of configurations, one for the classifier-specific
     hyper-parameters, and another for que quantifier-specific hyper-parameters
@@ -484,6 +534,7 @@
 
     return classifier_configs, quantifier_configs

 
+
 

 
            

diff --git a/docs/build/html/_modules/quapy/plot.html b/docs/build/html/_modules/quapy/plot.html
index 79179a1..4c0b05d 100644
--- a/docs/build/html/_modules/quapy/plot.html
+++ b/docs/build/html/_modules/quapy/plot.html
@@ -1,22 +1,20 @@
+
+
 
 
 
   
   
-  quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -42,7 +40,13 @@
 

         

@@ -70,16 +74,16 @@
            

              
   
Source code for quapy.plot
-from collections import defaultdict
-import matplotlib.pyplot as plt
-from matplotlib.cm import get_cmap
-import numpy as np
-from matplotlib import cm
-from scipy.stats import ttest_ind_from_stats
-from matplotlib.ticker import ScalarFormatter
-import math
+from collections import defaultdict
+import matplotlib.pyplot as plt
+from matplotlib.pyplot import get_cmap
+import numpy as np
+from matplotlib import cm
+from scipy.stats import ttest_ind_from_stats
+from matplotlib.ticker import ScalarFormatter
+import math
 
-import quapy as qp
+import quapy as qp
 
 plt.rcParams['figure.figsize'] = [10, 6]
 plt.rcParams['figure.dpi'] = 200
@@ -88,7 +92,7 @@
 
 

 [docs]
-def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
+def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
                     train_prev=None, savepath=None, method_order=None):
     """
     The diagonal plot displays the predicted prevalence values (along the y-axis) as a function of the true prevalence
@@ -97,21 +101,29 @@
     indicating which class is to be taken as the positive class. (For multiclass quantification problems, other plots
     like the :meth:`error_by_drift` might be preferable though).
 
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
+
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
-    :param pos_class: index of the positive class
-    :param title: the title to be displayed in the plot
-    :param show_std: whether or not to show standard deviations (represented by color bands). This might be inconvenient
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
+    :param pos_class: index of the positive class (default 1)
+    :param title: the title to be displayed in the plot (default None)
+    :param show_std: whether to show standard deviations (represented by color bands). This might be inconvenient
         for cases in which many methods are compared, or when the standard deviations are high -- default True)
-    :param legend: whether or not to display the leyend (default True)
-    :param train_prev: if indicated (default is None), the training prevalence (for the positive class) is hightlighted
-        in the plot. This is convenient when all the experiments have been conducted in the same dataset.
+    :param legend: whether to display the legend (default True)
+    :param train_prev: if indicated (default is None), the training prevalence (for the positive class) is highlighted
+        in the plot. This is convenient when all the experiments have been conducted in the same dataset, or in
+        datasets with the same training prevalence.
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
     :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
         listed in the legend and associated with matplotlib colors).
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
     fig, ax = plt.subplots()
     ax.set_aspect('equal')
@@ -152,31 +164,34 @@
 
     if legend:
         ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
-        # box = ax.get_position()
-        # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
-        # ax.legend(loc='lower center',
-        #           bbox_to_anchor=(1, -0.5),
-        #           ncol=(len(method_names)+1)//2)
 
-    _save_or_show(savepath)

+    _save_or_show(savepath)
+    return fig, ax

 
 
 
 

 [docs]
-def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None):
+def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None):
     """
     Box-plots displaying the global bias (i.e., signed error computed as the estimated value minus the true value)
     for each quantification method with respect to a given positive class.
 
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
+
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
     :param pos_class: index of the positive class
-    :param title: the title to be displayed in the plot
+    :param title: the title to be displayed in the plot (default None)
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
 
     method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs)
@@ -195,32 +210,41 @@
     plt.xticks(rotation=45)
     ax.set(ylabel='error bias', title=title)
 
-    _save_or_show(savepath)

+    _save_or_show(savepath)
+
+    return fig, ax

 
 
 
 

 [docs]
-def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10,
+def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10,
                      vertical_xticks=False, legend=True, savepath=None):
     """
     Box-plots displaying the local bias (i.e., signed error computed as the estimated value minus the true value)
-    for different bins of (true) prevalence of the positive classs, for each quantification method.
+    for different bins of (true) prevalence of the positive class, for each quantification method.
+
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
 
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
     :param pos_class: index of the positive class
-    :param title: the title to be displayed in the plot
-    :param nbins: number of bins
+    :param title: the title to be displayed in the plot (default None)
+    :param nbins: number of bins (default 5)
     :param colormap: the matplotlib colormap to use (default cm.tab10)
     :param vertical_xticks: whether or not to add secondary grid (default is False)
     :param legend: whether or not to display the legend (default is True)
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
-    from pylab import boxplot, plot, setp
+    from pylab import boxplot, plot, setp
 
     fig, ax = plt.subplots()
     ax.grid()
@@ -288,18 +312,20 @@
     ax.set(xlabel='prevalence', ylabel='error bias', title=title)
     ax.set_xlim(0, 1)
 
-    _save_or_show(savepath)

+    _save_or_show(savepath)
+
+    return fig, ax

 
 
 
 

 [docs]
-def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
+def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
                    n_bins=20, error_name='ae', show_std=False,
                    show_density=True,
                    show_legend=True,
                    logscale=False,
-                   title=f'Quantification error as a function of distribution shift',
+                   title=None,
                    vlines=None,
                    method_order=None,
                    savepath=None):
@@ -310,11 +336,17 @@
     fare in different regions of the prior probability shift spectrum (e.g., in the low-shift regime vs. in the
     high-shift regime).
 
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
+
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
     :param tr_prevs: training prevalence of each experiment
     :param n_bins: number of bins in which the y-axis is to be divided (default is 20)
     :param error_name: a string representing the name of an error function (as defined in `quapy.error`, default is "ae")
@@ -322,12 +354,13 @@
     :param show_density: whether or not to display the distribution of experiments for each bin (default is True)
     :param show_density: whether or not to display the legend of the chart (default is True)
     :param logscale: whether or not to log-scale the y-error measure (default is False)
-    :param title: title of the plot (default is "Quantification error as a function of distribution shift")
+    :param title: title of the plot (default is None)
     :param vlines: array-like list of values (default is None). If indicated, highlights some regions of the space
         using vertical dotted lines.
     :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
         listed in the legend and associated with matplotlib colors).
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
 
     fig, ax = plt.subplots()
@@ -336,14 +369,14 @@
     x_error = qp.error.ae
     y_error = getattr(qp.error, error_name)
 
+    if method_order is None:
+        method_order = []
+
     # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same
     # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to
     # x_error function) and 'y' is the estim-test shift (computed as according to y_error)
     data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order)
 
-    if method_order is None:
-        method_order = method_names
-
     _set_colors(ax, n_methods=len(method_order))
 
     bins = np.linspace(0, 1, n_bins+1)
@@ -396,11 +429,11 @@
         ax2.spines['right'].set_color('g')
         ax2.tick_params(axis='y', colors='g')
     
-    ax.set(xlabel=f'Distribution shift between training set and test sample',
-           ylabel=f'{error_name.upper()} (true distribution, predicted distribution)',
+    ax.set(xlabel=f'Prior shift between training set and test sample',
+           ylabel=f'{error_name.upper()} (true prev, predicted prev)',
            title=title)
-    box = ax.get_position()
-    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
+    # box = ax.get_position()
+    # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
     if vlines:
         for vline in vlines:
             ax.axvline(vline, 0, 1, linestyle='--', color='k')
@@ -410,19 +443,20 @@
         #nice scale for the logaritmic axis
         ax.set_ylim(0,10 ** math.ceil(math.log10(max_y)))
     
-    
     if show_legend:
-        fig.legend(loc='lower center',
+        fig.legend(loc='center left',
                   bbox_to_anchor=(1, 0.5),
-                  ncol=(len(method_names)+1)//2)
-      
-    _save_or_show(savepath)

+                  ncol=1)
+
+    _save_or_show(savepath)
+
+    return fig, ax

 
 
 
 

 [docs]
-def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
+def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
                                  n_bins=20, binning='isomerous',
                                  x_error='ae', y_error='ae', ttest_alpha=0.005, tail_density_threshold=0.005,
                                  method_order=None,
@@ -436,11 +470,17 @@
     plot is displayed on top, that displays the distribution of experiments for each bin (when binning="isometric") or
     the percentiles points of the distribution (when binning="isomerous").
 
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
+
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
     :param tr_prevs: training prevalence of each experiment
     :param n_bins: number of bins in which the y-axis is to be divided (default is 20)
     :param binning: type of binning, either "isomerous" (default) or "isometric"
@@ -457,13 +497,16 @@
     :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
         listed in the legend and associated with matplotlib colors).
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
-    :return:
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
     assert binning in ['isomerous', 'isometric'], 'unknown binning type; valid types are "isomerous" and "isometric"'
 
     x_error = getattr(qp.error, x_error)
     y_error = getattr(qp.error, y_error)
 
+    if method_order is None:
+        method_order = []
+
     # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same
     # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to
     # x_error function) and 'y' is the estim-test shift (computed as according to y_error)
@@ -602,11 +645,13 @@
         ax.get_xaxis().set_visible(False)
         plt.subplots_adjust(wspace=0, hspace=0)
 
-    _save_or_show(savepath)

+    _save_or_show(savepath)
+
+    return fig, ax

 
 
 
-def _merge(method_names, true_prevs, estim_prevs):
+def _merge(method_names, true_prevs, estim_prevs):
     ndims = true_prevs[0].shape[1]
     data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))})
     method_order=[]
@@ -620,13 +665,14 @@
     return method_order, true_prevs_, estim_prevs_
 
 
-def _set_colors(ax, n_methods):
+def _set_colors(ax, n_methods):
     NUM_COLORS = n_methods
-    cm = plt.get_cmap('tab20')
-    ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)])
+    if NUM_COLORS>10:
+        cm = plt.get_cmap('tab20')
+        ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)])
 
 
-def _save_or_show(savepath):
+def _save_or_show(savepath):
     # if savepath is specified, then saves the plot in that path; otherwise the plot is shown
     if savepath is not None:
         qp.util.create_parent_dir(savepath)
@@ -636,7 +682,7 @@
         plt.show()
 
 
-def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order):
+def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order):
     data = defaultdict(lambda: {'x': np.empty(shape=(0)), 'y': np.empty(shape=(0))})
 
     if method_order is None:
@@ -655,6 +701,45 @@
             method_order.append(method)
 
     return data
+
+
+

+[docs]
+def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None):
+    posteriors = prob_classifier.predict_proba(X)
+    assert posteriors.ndim==2, 'calibration plot only works for binary problems'
+    posteriors = posteriors[:,1]
+    pred_y = posteriors>=0.5
+    bins = np.linspace(0, 1, nbins + 1)
+    binned_values = np.digitize(posteriors, bins, right=False)
+    print(np.unique(binned_values))
+    correct = pred_y == y
+    bin_centers = (bins[:-1] + bins[1:]) / 2
+    bins_names = np.arange(nbins)
+    y_axis = [correct[binned_values==bin].mean() for bin in bins_names]
+    y_axis = [v if not np.isnan(v) else 0 for v in y_axis]
+    # Crear el gráfico de barras
+    plt.bar(bin_centers, y_axis, width=bins[1]-bins[0], edgecolor='black', alpha=0.7)
+
+    # Etiquetas y título
+    plt.xlabel("Bin")
+    plt.ylabel("Value")
+    plt.title("Bar plot of calculated values per bin")
+    plt.xticks(bin_centers, [f"{b:.2f}" for b in bin_centers], rotation=45)
+
+    # Mostrar el gráfico
+    plt.tight_layout()
+    plt.show()

+
+
+if __name__ == '__main__':
+    import quapy as qp
+    from sklearn.linear_model import LogisticRegression
+    data = qp.datasets.fetch_UCIBinaryDataset(qp.datasets.UCI_BINARY_DATASETS[6])
+    train, test = data.train_test
+    classifier = LogisticRegression()
+    classifier.fit(*train.Xy)
+    calibration_plot(classifier, *test.Xy)
diff --git a/docs/build/html/_modules/quapy/protocol.html b/docs/build/html/_modules/quapy/protocol.html index 7d96338..50e3cb8 100644 --- a/docs/build/html/_modules/quapy/protocol.html +++ b/docs/build/html/_modules/quapy/protocol.html @@ -1,23 +1,20 @@ + + - + - quapy.protocol — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.protocol — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@
@@ -71,25 +74,31 @@

Source code for quapy.protocol

-from copy import deepcopy
-import quapy as qp
-import numpy as np
-import itertools
-from contextlib import ExitStack
-from abc import ABCMeta, abstractmethod
-from quapy.data import LabelledCollection
-import quapy.functional as F
-from os.path import exists
-from glob import glob
+from copy import deepcopy
+from typing import Iterable
+
+import quapy as qp
+import numpy as np
+import itertools
+from contextlib import ExitStack
+from abc import ABCMeta, abstractmethod
+from quapy.data import LabelledCollection
+import quapy.functional as F
+from os.path import exists
+from glob import glob
+from collections.abc import Iterable
+from numbers import Number
 
 
-
[docs]class AbstractProtocol(metaclass=ABCMeta):
+

+[docs]
+class AbstractProtocol(metaclass=ABCMeta):
     """
     Abstract parent class for sample generation protocols.
     """
 
     @abstractmethod
-    def __call__(self):
+    def __call__(self):
         """
         Implements the protocol. Yields one sample at a time along with its prevalence
 
@@ -98,25 +107,31 @@
         """
         ...
 
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Indicates the total number of samples that the protocol generates.
 
         :return: The number of samples to generate if known, or `None` otherwise.
         """
-        return None

+        return None

+

 
 
-
[docs]class IterateProtocol(AbstractProtocol):
+
+

+[docs]
+class IterateProtocol(AbstractProtocol):
     """
     A very simple protocol which simply iterates over a list of previously generated samples
 
     :param samples: a list of :class:`quapy.data.base.LabelledCollection`
     """
-    def __init__(self, samples: [LabelledCollection]):
+    def __init__(self, samples: [LabelledCollection]):
         self.samples = samples
 
-    def __call__(self):
+    def __call__(self):
         """
         Yields one sample from the initial list at a time
 
@@ -126,16 +141,58 @@
         for sample in self.samples:
             yield sample.Xp
 
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples in this protocol
 
         :return: int
         """
-        return len(self.samples)

+        return len(self.samples)

+

 
 
-
[docs]class AbstractStochasticSeededProtocol(AbstractProtocol):
+
+

+[docs]
+class ProtocolFromIndex(AbstractProtocol):
+    """
+    A protocol from a list of indexes
+
+    :param data: a :class:`quapy.data.base.LabelledCollection`
+    :param indexes: a list of indexes
+    """
+    def __init__(self, data: LabelledCollection, indexes: Iterable):
+        self.data = data
+        self.indexes = indexes
+
+    def __call__(self):
+        """
+        Yields one sample at a time extracted using the indexes
+
+        :return: yields a tuple `(sample, prev) at a time, where `sample` is a set of instances
+            and in which `prev` is an `nd.array` with the class prevalence values
+        """
+        for index in self.indexes:
+            yield self.data.sampling_from_index(index).Xp
+
+

+[docs]
+    def total(self):
+        """
+        Returns the number of samples in this protocol
+
+        :return: int
+        """
+        return len(self.indexes)

+

+
+
+
+

+[docs]
+class AbstractStochasticSeededProtocol(AbstractProtocol):
     """
     An `AbstractStochasticSeededProtocol` is a protocol that generates, via any random procedure (e.g.,
     via random sampling), sequences of :class:`quapy.data.base.LabelledCollection` samples.
@@ -152,19 +209,21 @@
 
     _random_state = -1  # means "not set"
 
-    def __init__(self, random_state=0):
+    def __init__(self, random_state=0):
         self.random_state = random_state
 
     @property
-    def random_state(self):
+    def random_state(self):
         return self._random_state
 
     @random_state.setter
-    def random_state(self, random_state):
+    def random_state(self, random_state):
         self._random_state = random_state
 
-
[docs]    @abstractmethod
-    def samples_parameters(self):
+

+[docs]
+    @abstractmethod
+    def samples_parameters(self):
         """
         This function has to return all the necessary parameters to replicate the samples
 
@@ -172,8 +231,11 @@
         """
         ...

 
-
[docs]    @abstractmethod
-    def sample(self, params):
+
+

+[docs]
+    @abstractmethod
+    def sample(self, params):
         """
         Extract one sample determined by the given parameters
 
@@ -182,7 +244,8 @@
         """
         ...

 
-    def __call__(self):
+
+    def __call__(self):
         """
         Yields one sample at a time. The type of object returned depends on the `collator` function. The
         default behaviour returns tuples of the form `(sample, prevalence)`.
@@ -197,9 +260,11 @@
             if self.random_state is not None:
                 stack.enter_context(qp.util.temp_seed(self.random_state))
             for params in self.samples_parameters():
-                yield self.collator(self.sample(params))
+                yield self.collator(self.sample(params), params)
 
-
[docs]    def collator(self, sample, *args):
+

+[docs]
+    def collator(self, sample, params):
         """
         The collator prepares the sample to accommodate the desired output format before returning the output.
         This collator simply returns the sample as it is. Classes inheriting from this abstract class can
@@ -209,17 +274,23 @@
         :param args: additional arguments
         :return: the sample adhering to a desired output format (in this case, the sample is returned as it is)
         """
-        return sample

+        return sample

+

 
 
-
[docs]class OnLabelledCollectionProtocol:
+
+

+[docs]
+class OnLabelledCollectionProtocol(AbstractStochasticSeededProtocol):
     """
     Protocols that generate samples from a :class:`qp.data.LabelledCollection` object.
     """
 
     RETURN_TYPES = ['sample_prev', 'labelled_collection', 'index']
 
-
[docs]    def get_labelled_collection(self):
+

+[docs]
+    def get_labelled_collection(self):
         """
         Returns the labelled collection on which this protocol acts.
 
@@ -227,7 +298,10 @@
         """
         return self.data

 
-
[docs]    def on_preclassified_instances(self, pre_classifications, in_place=False):
+
+

+[docs]
+    def on_preclassified_instances(self, pre_classifications, in_place=False):
         """
         Returns a copy of this protocol that acts on a modified version of the original
         :class:`qp.data.LabelledCollection` in which the original instances have been replaced
@@ -250,8 +324,11 @@
             new = deepcopy(self)
             return new.on_preclassified_instances(pre_classifications, in_place=True)

 
-
[docs]    @classmethod
-    def get_collator(cls, return_type='sample_prev'):
+
+

+[docs]
+    @classmethod
+    def get_collator(cls, return_type='sample_prev'):
         """
         Returns a collator function, i.e., a function that prepares the yielded data
 
@@ -264,12 +341,30 @@
         assert return_type in cls.RETURN_TYPES, \
             f'unknown return type passed as argument; valid ones are {cls.RETURN_TYPES}'
         if return_type=='sample_prev':
-            return lambda lc:lc.Xp
+            return lambda lc,params:lc.Xp
         elif return_type=='labelled_collection':
-            return lambda lc:lc

+            return lambda lc,params:lc
+        elif return_type=='index':
+            return lambda lc,params:params

 
 
-
[docs]class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+

+[docs]
+    def sample(self, index):
+        """
+        Realizes the sample given the index of the instances.
+
+        :param index: indexes of the instances to select
+        :return: an instance of :class:`qp.data.LabelledCollection`
+        """
+        return self.data.sampling_from_index(index)

+

+
+
+
+

+[docs]
+class APP(OnLabelledCollectionProtocol):
     """
     Implementation of the artificial prevalence protocol (APP).
     The APP consists of exploring a grid of prevalence values containing `n_prevalences` points (e.g.,
@@ -293,7 +388,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data: LabelledCollection, sample_size=None, n_prevalences=21, repeats=10,
+    def __init__(self, data: LabelledCollection, sample_size=None, n_prevalences=21, repeats=10,
                  smooth_limits_epsilon=0, random_state=0, sanity_check=10000, return_type='sample_prev'):
         super(APP, self).__init__(random_state)
         self.data = data
@@ -313,7 +408,9 @@
 
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def prevalence_grid(self):
+

+[docs]
+    def prevalence_grid(self):
         """
         Generates vectors of prevalence values from an exhaustive grid of prevalence values. The
         number of prevalence values explored for each dimension depends on `n_prevalences`, so that, if, for example,
@@ -338,7 +435,10 @@
             prevs = np.repeat(prevs, self.repeats, axis=0)
         return prevs

 
-
[docs]    def samples_parameters(self):
+
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the APP protocol.
 
@@ -350,25 +450,23 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated
 
         :return: int
         """
-        return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats)

+        return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats)

+

 
 
-
[docs]class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+
+

+[docs]
+class NPP(OnLabelledCollectionProtocol):
     """
     A generator of samples that implements the natural prevalence protocol (NPP). The NPP consists of drawing
     samples uniformly at random, therefore approximately preserving the natural prevalence of the collection.
@@ -383,7 +481,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data:LabelledCollection, sample_size=None, repeats=100, random_state=0,
+    def __init__(self, data:LabelledCollection, sample_size=None, repeats=100, random_state=0,
                  return_type='sample_prev'):
         super(NPP, self).__init__(random_state)
         self.data = data
@@ -392,7 +490,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the NPP protocol.
 
@@ -404,25 +504,23 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats")
 
         :return: int
         """
-        return self.repeats

+        return self.repeats

+

 
 
-
[docs]class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+
+

+[docs]
+class UPP(OnLabelledCollectionProtocol):
     """
     A variant of :class:`APP` that, instead of using a grid of equidistant prevalence values,
     relies on the Kraemer algorithm for sampling unit (k-1)-simplex uniformly at random, with
@@ -441,7 +539,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data: LabelledCollection, sample_size=None, repeats=100, random_state=0,
+    def __init__(self, data: LabelledCollection, sample_size=None, repeats=100, random_state=0,
                  return_type='sample_prev'):
         super(UPP, self).__init__(random_state)
         self.data = data
@@ -450,7 +548,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the UPP protocol.
 
@@ -462,25 +562,96 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats")
 
         :return: int
         """
-        return self.repeats

+        return self.repeats

+

 
 
-
[docs]class DomainMixer(AbstractStochasticSeededProtocol):
+
+

+[docs]
+class DirichletProtocol(OnLabelledCollectionProtocol):
+    """
+    A protocol that establishes a prior Dirichlet distribution for the prevalence of the samples.
+    Note that providing an all-ones vector of Dirichlet parameters is equivalent to invoking the
+    APP protocol (although each protocol will generate a different series of samples given a
+    fixed seed, since the implementation is different).
+
+    :param data: a `LabelledCollection` from which the samples will be drawn
+    :param alpha: an array-like of shape (n_classes,) with the parameters of the Dirichlet distribution
+    :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from
+        qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised.
+    :param repeats: the number of samples to generate. Default is 100.
+    :param random_state: allows replicating samples across runs (default 0, meaning that the sequence of samples
+        will be the same every time the protocol is called)
+    :param return_type: set to "sample_prev" (default) to get the pairs of (sample, prevalence) at each iteration, or
+        to "labelled_collection" to get instead instances of LabelledCollection
+    """
+
+    def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0,
+                 return_type='sample_prev'):
+        #assert ((isinstance(alpha, str) and alpha == 'uniform') or
+        #        isinstance(alpha, Number) or
+        #        (isinstance(alpha, Iterable) and all(isinstance(v, Number) for v in alpha))), \
+        #    f'wrong type for {alpha=}; expected "uniform", a real scalar, or an array-like of real values'
+        
+        n_classes = data.n_classes
+        if isinstance(alpha, str) and alpha == 'uniform':
+            self.alpha = np.ones(n_classes, dtype=float)
+        elif isinstance(alpha, Number):
+            self.alpha = np.full(n_classes, float(alpha), dtype=float)
+        else:
+            self.alpha = np.asarray(alpha, dtype=float)
+            if self.alpha.ndim != 1 or len(self.alpha) != n_classes:
+                raise ValueError(
+                    f'wrong shape for alpha; expected {n_classes} values, found shape {self.alpha.shape}'
+                )
+
+        super(DirichletProtocol, self).__init__(random_state)
+        self.data = data
+        #self.alpha = alpha
+        self.sample_size = qp._get_sample_size(sample_size)
+        self.repeats = repeats
+        self.random_state = random_state
+        self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
+
+

+[docs]
+    def samples_parameters(self):
+        """
+        Return all the necessary parameters to replicate the samples.
+
+        :return: a list of indexes that realize the sampling
+        """
+        prevs = np.random.dirichlet(self.alpha, size=self.repeats)
+        indexes = [self.data.sampling_index(self.sample_size, *prevs_i) for prevs_i in prevs]
+        return indexes

+
+
+

+[docs]
+    def total(self):
+        """
+        Returns the number of samples that will be generated (equals to "repeats")
+
+        :return: int
+        """
+        return self.repeats

+

+
+
+
+

+[docs]
+class DomainMixer(AbstractStochasticSeededProtocol):
     """
     Generates mixtures of two domains (A and B) at controlled rates, but preserving the original class prevalence.
 
@@ -489,7 +660,7 @@
     :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from
         qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised.
     :param repeats: int, number of samples to draw for every mixture rate
-    :param prevalence: the prevalence to preserv along the mixtures. If specified, should be an array containing
+    :param prevalence: the prevalence to preserve along the mixtures. If specified, should be an array containing
         one prevalence value (positive float) for each class and summing up to one. If not specified, the prevalence
         will be taken from the domain A (default).
     :param mixture_points: an integer indicating the number of points to take from a linear scale (e.g., 21 will
@@ -499,7 +670,7 @@
         will be the same every time the protocol is called)
     """
 
-    def __init__(
+    def __init__(
             self,
             domainA: LabelledCollection,
             domainB: LabelledCollection,
@@ -531,7 +702,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the this protocol.
 
@@ -548,7 +721,10 @@
                 indexesB.append(sampleBidx)
         return list(zip(indexesA, indexesB))

 
-
[docs]    def sample(self, indexes):
+
+

+[docs]
+    def sample(self, indexes):
         """
         Realizes the sample given a pair of indexes of the instances from A and B.
 
@@ -560,13 +736,18 @@
         sampleB = self.B.sampling_from_index(indexesB)
         return sampleA+sampleB

 
-
[docs]    def total(self):
+
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats * mixture_points")
 
         :return: int
         """
-        return self.repeats * len(self.mixture_points)

+        return self.repeats * len(self.mixture_points)

+

+
 
 
 # aliases
diff --git a/docs/build/html/_modules/quapy/util.html b/docs/build/html/_modules/quapy/util.html
index 25532bd..5f34290 100644
--- a/docs/build/html/_modules/quapy/util.html
+++ b/docs/build/html/_modules/quapy/util.html
@@ -1,23 +1,20 @@
+
+
 
-
+
 
   
   
-  quapy.util — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-      
-      
+  quapy.util — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+      
+      
 
   
-  
-  
-        
-        
-        
-        
-        
-        
+      
+      
+      
+      
+      
     
     
      
@@ -43,7 +40,13 @@
 

         

@@ -71,23 +74,26 @@
            

              
   
Source code for quapy.util
-import contextlib
-import itertools
-import multiprocessing
-import os
-import pickle
-import urllib
-from pathlib import Path
-from contextlib import ExitStack
-import quapy as qp
+import contextlib
+import itertools
+import multiprocessing
+import os
+import pickle
+import urllib
+from pathlib import Path
+from contextlib import ExitStack
 
-import numpy as np
-from joblib import Parallel, delayed
-from time import time
-import signal
+import pandas as pd
+
+import quapy as qp
+
+import numpy as np
+from joblib import Parallel, delayed
+from time import time
+import signal
 
 
-def _get_parallel_slices(n_tasks, n_jobs):
+def _get_parallel_slices(n_tasks, n_jobs):
     if n_jobs == -1:
         n_jobs = multiprocessing.cpu_count()
     batch = int(n_tasks / n_jobs)
@@ -95,7 +101,9 @@
     return [slice(job * batch, (job + 1) * batch + (remainder if job == n_jobs - 1 else 0)) for job in range(n_jobs)]
 
 
-
[docs]def map_parallel(func, args, n_jobs):
+

+[docs]
+def map_parallel(func, args, n_jobs):
     """
     Applies func to n_jobs slices of args. E.g., if args is an array of 99 items and n_jobs=2, then
     func is applied in two parallel processes to args[0:50] and to args[50:99]. func is a function
@@ -113,7 +121,10 @@
     return list(itertools.chain.from_iterable(results))

 
 
-
[docs]def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
+
+

+[docs]
+def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
     """
     A wrapper of multiprocessing:
 
@@ -129,8 +140,9 @@
     :param seed: the numeric seed
     :param asarray: set to True to return a np.ndarray instead of a list
     :param backend: indicates the backend used for handling parallel works
+    :param open_args: if True, then the delayed function is called on *args_i, instead of on args_i
     """
-    def func_dec(environ, seed, *args):
+    def func_dec(environ, seed, *args):
         qp.environ = environ.copy()
         qp.environ['N_JOBS'] = 1
         #set a context with a temporal seed to ensure results are reproducibles in parallel
@@ -147,8 +159,48 @@
     return out

 
 
-
[docs]@contextlib.contextmanager
-def temp_seed(random_state):
+
+

+[docs]
+def parallel_unpack(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
+    """
+    A wrapper of multiprocessing:
+
+    >>> Parallel(n_jobs=n_jobs)(
+    >>>      delayed(func)(*args_i) for args_i in args
+    >>> )
+
+    that takes the `quapy.environ` variable as input silently.
+    Seeds the child processes to ensure reproducibility when n_jobs>1.
+
+    :param func: callable
+    :param args: args of func
+    :param seed: the numeric seed
+    :param asarray: set to True to return a np.ndarray instead of a list
+    :param backend: indicates the backend used for handling parallel works
+    """
+
+    def func_dec(environ, seed, *args):
+        qp.environ = environ.copy()
+        qp.environ['N_JOBS'] = 1
+        # set a context with a temporal seed to ensure results are reproducibles in parallel
+        with ExitStack() as stack:
+            if seed is not None:
+                stack.enter_context(qp.util.temp_seed(seed))
+            return func(*args)
+
+    out = Parallel(n_jobs=n_jobs, backend=backend)(
+        delayed(func_dec)(qp.environ, None if seed is None else seed + i, *args_i) for i, args_i in enumerate(args)
+    )
+    if asarray:
+        out = np.asarray(out)
+    return out

+
+
+

+[docs]
+@contextlib.contextmanager
+def temp_seed(random_state):
     """
     Can be used in a "with" context to set a temporal seed without modifying the outer numpy's current state. E.g.:
 
@@ -169,14 +221,17 @@
             np.random.set_state(state)

 
 
-
[docs]def download_file(url, archive_filename):
+
+

+[docs]
+def download_file(url, archive_filename):
     """
     Downloads a file from a url
 
     :param url: the url
     :param archive_filename: destination filename
     """
-    def progress(blocknum, bs, size):
+    def progress(blocknum, bs, size):
         total_sz_mb = '%.2f MB' % (size / 1e6)
         current_sz_mb = '%.2f MB' % ((blocknum * bs) / 1e6)
         print('\rdownloaded %s / %s' % (current_sz_mb, total_sz_mb), end='')
@@ -185,7 +240,10 @@
     print("")

 
 
-
[docs]def download_file_if_not_exists(url, archive_filename):
+
+

+[docs]
+def download_file_if_not_exists(url, archive_filename):
     """
     Dowloads a function (using :meth:`download_file`) if the file does not exist.
 
@@ -198,7 +256,10 @@
     download_file(url, archive_filename)

 
 
-
[docs]def create_if_not_exist(path):
+
+

+[docs]
+def create_if_not_exist(path):
     """
     An alias to `os.makedirs(path, exist_ok=True)` that also returns the path. This is useful in cases like, e.g.:
 
@@ -211,7 +272,10 @@
     return path

 
 
-
[docs]def get_quapy_home():
+
+

+[docs]
+def get_quapy_home():
     """
     Gets the home directory of QuaPy, i.e., the directory where QuaPy saves permanent data, such as dowloaded datasets.
     This directory is `~/quapy_data`
@@ -223,7 +287,10 @@
     return home

 
 
-
[docs]def create_parent_dir(path):
+
+

+[docs]
+def create_parent_dir(path):
     """
     Creates the parent dir (if any) of a given path, if not exists. E.g., for `./path/to/file.txt`, the path `./path/to`
     is created.
@@ -235,7 +302,10 @@
         os.makedirs(parentdir, exist_ok=True)

 
 
-
[docs]def save_text_file(path, text):
+
+

+[docs]
+def save_text_file(path, text):
     """
     Saves a text file to disk, given its full path, and creates the parent directory if missing.
 
@@ -243,11 +313,14 @@
     :param text: text to save.
     """
     create_parent_dir(path)
-    with open(text, 'wt') as fout:
+    with open(path, 'wt') as fout:
         fout.write(text)

 
 
-
[docs]def pickled_resource(pickle_path:str, generation_func:callable, *args):
+
+

+[docs]
+def pickled_resource(pickle_path:str, generation_func:callable, *args):
     """
     Allows for fast reuse of resources that are generated only once by calling generation_func(\\*args). The next times
     this function is invoked, it loads the pickled resource. Example:
@@ -266,15 +339,18 @@
         return generation_func(*args)
     else:
         if os.path.exists(pickle_path):
-            return pickle.load(open(pickle_path, 'rb'))
+            with open(pickle_path, 'rb') as fin:
+                instance = pickle.load(fin)
         else:
             instance = generation_func(*args)
             os.makedirs(str(Path(pickle_path).parent), exist_ok=True)
-            pickle.dump(instance, open(pickle_path, 'wb'), pickle.HIGHEST_PROTOCOL)
-            return instance

+            with open(pickle_path, 'wb') as foo:
+                pickle.dump(instance, foo, pickle.HIGHEST_PROTOCOL)
+        return instance

 
 
-def _check_sample_size(sample_size):
+
+def _check_sample_size(sample_size):
     if sample_size is None:
         assert qp.environ['SAMPLE_SIZE'] is not None, \
             'error: sample_size set to None, and cannot be resolved from the environment'
@@ -284,7 +360,34 @@
     return sample_size
 
 
-
[docs]class EarlyStop:
+

+[docs]
+def load_report(path, as_dict=False):
+    def str2prev_arr(strprev):
+        within = strprev.strip('[]').split()
+        float_list = [float(p) for p in within]
+        float_list[-1] = 1. - sum(float_list[:-1])
+        return np.asarray(float_list)
+
+    df = pd.read_csv(path, index_col=0)
+    df['true-prev'] = df['true-prev'].apply(str2prev_arr)
+    df['estim-prev'] = df['estim-prev'].apply(str2prev_arr)
+    if as_dict:
+        d = {}
+        for col in df.columns.values:
+            vals = df[col].values
+            if col in ['true-prev', 'estim-prev']:
+                vals = np.vstack(vals)
+            d[col] = vals
+        return d
+    else:
+        return df

+
+
+
+

+[docs]
+class EarlyStop:
     """
     A class implementing the early-stopping condition typically used for training neural networks.
 
@@ -309,7 +412,7 @@
     :ivar IMPROVED: flag (boolean) indicating whether there was an improvement in the last call
     """
 
-    def __init__(self, patience, lower_is_better=True):
+    def __init__(self, patience, lower_is_better=True):
 
         self.PATIENCE_LIMIT = patience
         self.better = lambda a,b: a<b if lower_is_better else a>b
@@ -319,7 +422,7 @@
         self.STOP = False
         self.IMPROVED = False
 
-    def __call__(self, watch_score, epoch):
+    def __call__(self, watch_score, epoch):
         """
         Commits the new score found in epoch `epoch`. If the score improves over the best score found so far, then
         the patiente counter gets reset. If otherwise, the patience counter is decreased, and in case it reachs 0,
@@ -339,8 +442,11 @@
                 self.STOP = True

 
 
-
[docs]@contextlib.contextmanager
-def timeout(seconds):
+
+

+[docs]
+@contextlib.contextmanager
+def timeout(seconds):
     """
     Opens a context that will launch an exception if not closed after a given number of seconds
 
@@ -359,7 +465,7 @@
     :param seconds: number of seconds, set to <=0 to ignore the timer
     """
     if seconds > 0:
-        def handler(signum, frame):
+        def handler(signum, frame):
             raise TimeoutError()
 
         signal.signal(signal.SIGALRM, handler)
@@ -370,6 +476,7 @@
     if seconds > 0:
         signal.alarm(0)

 
+
 

 
            

diff --git a/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js b/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js
index 8549469..8141580 100644
--- a/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js
+++ b/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js
@@ -1,20 +1,9 @@
-/*
- * _sphinx_javascript_frameworks_compat.js
- * ~~~~~~~~~~
- *
- * Compatability shim for jQuery and underscores.js.
- *
- * WILL BE REMOVED IN Sphinx 6.0
- * xref RemovedInSphinx60Warning
+/* Compatability shim for jQuery and underscores.js.
  *
+ * Copyright Sphinx contributors
+ * Released under the two clause BSD licence
  */
 
-/**
- * select a different prefix for underscore
- */
-$u = _.noConflict();
-
-
 /**
  * small helper function to urldecode strings
  *
diff --git a/docs/build/html/_static/css/badge_only.css b/docs/build/html/_static/css/badge_only.css
index c718cee..88ba55b 100644
--- a/docs/build/html/_static/css/badge_only.css
+++ b/docs/build/html/_static/css/badge_only.css
@@ -1 +1 @@
-.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
\ No newline at end of file
+.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}
\ No newline at end of file
diff --git a/docs/build/html/_static/css/theme.css b/docs/build/html/_static/css/theme.css
index 19a446a..a88467c 100644
--- a/docs/build/html/_static/css/theme.css
+++ b/docs/build/html/_static/css/theme.css
@@ -1,4 +1,4 @@
 html{box-sizing:border-box}*,:after,:before{box-sizing:inherit}article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}[hidden],audio:not([controls]){display:none}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}blockquote{margin:0}dfn{font-style:italic}ins{background:#ff9;text-decoration:none}ins,mark{color:#000}mark{background:#ff0;font-style:italic;font-weight:700}.rst-content code,.rst-content tt,code,kbd,pre,samp{font-family:monospace,serif;_font-family:courier new,monospace;font-size:1em}pre{white-space:pre}q{quotes:none}q:after,q:before{content:"";content:none}small{font-size:85%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}dl,ol,ul{margin:0;padding:0;list-style:none;list-style-image:none}li{list-style:none}dd{margin:0}img{border:0;-ms-interpolation-mode:bicubic;vertical-align:middle;max-width:100%}svg:not(:root){overflow:hidden}figure,form{margin:0}label{cursor:pointer}button,input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}button,input{line-height:normal}button,input[type=button],input[type=reset],input[type=submit]{cursor:pointer;-webkit-appearance:button;*overflow:visible}button[disabled],input[disabled]{cursor:default}input[type=search]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}textarea{resize:vertical}table{border-collapse:collapse;border-spacing:0}td{vertical-align:top}.chromeframe{margin:.2em 0;background:#ccc;color:#000;padding:.2em 0}.ir{display:block;border:0;text-indent:-999em;overflow:hidden;background-color:transparent;background-repeat:no-repeat;text-align:left;direction:ltr;*line-height:0}.ir br{display:none}.hidden{display:none!important;visibility:hidden}.visuallyhidden{border:0;clip:rect(0 0 0 0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.visuallyhidden.focusable:active,.visuallyhidden.focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.invisible{visibility:hidden}.relative{position:relative}big,small{font-size:100%}@media print{body,html,section{background:none!important}*{box-shadow:none!important;text-shadow:none!important;filter:none!important;-ms-filter:none!important}a,a:visited{text-decoration:underline}.ir a:after,a[href^="#"]:after,a[href^="javascript:"]:after{content:""}blockquote,pre{page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}img{max-width:100%!important}@page{margin:.5cm}.rst-content .toctree-wrapper>p.caption,h2,h3,p{orphans:3;widows:3}.rst-content .toctree-wrapper>p.caption,h2,h3{page-break-after:avoid}}.btn,.fa:before,.icon:before,.rst-content .admonition,.rst-content .admonition-title:before,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .code-block-caption .headerlink:before,.rst-content .danger,.rst-content .eqno .headerlink:before,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-alert,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before,input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week],select,textarea{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}/*!
  *  Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome
  *  License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License)
- */@font-face{font-family:FontAwesome;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713);src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix&v=4.7.0) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#fontawesomeregular) format("svg");font-weight:400;font-style:normal}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14286em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14286em;width:2.14286em;top:.14286em;text-align:center}.fa-li.fa-lg{left:-1.85714em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa-pull-left.icon,.fa.fa-pull-left,.rst-content .code-block-caption .fa-pull-left.headerlink,.rst-content .eqno .fa-pull-left.headerlink,.rst-content .fa-pull-left.admonition-title,.rst-content code.download span.fa-pull-left:first-child,.rst-content dl dt .fa-pull-left.headerlink,.rst-content h1 .fa-pull-left.headerlink,.rst-content h2 .fa-pull-left.headerlink,.rst-content h3 .fa-pull-left.headerlink,.rst-content h4 .fa-pull-left.headerlink,.rst-content h5 .fa-pull-left.headerlink,.rst-content h6 .fa-pull-left.headerlink,.rst-content p .fa-pull-left.headerlink,.rst-content table>caption .fa-pull-left.headerlink,.rst-content tt.download span.fa-pull-left:first-child,.wy-menu-vertical li.current>a button.fa-pull-left.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-left.toctree-expand,.wy-menu-vertical li button.fa-pull-left.toctree-expand{margin-right:.3em}.fa-pull-right.icon,.fa.fa-pull-right,.rst-content .code-block-caption .fa-pull-right.headerlink,.rst-content .eqno .fa-pull-right.headerlink,.rst-content .fa-pull-right.admonition-title,.rst-content code.download span.fa-pull-right:first-child,.rst-content dl dt .fa-pull-right.headerlink,.rst-content h1 .fa-pull-right.headerlink,.rst-content h2 .fa-pull-right.headerlink,.rst-content h3 .fa-pull-right.headerlink,.rst-content h4 .fa-pull-right.headerlink,.rst-content h5 .fa-pull-right.headerlink,.rst-content h6 .fa-pull-right.headerlink,.rst-content p .fa-pull-right.headerlink,.rst-content table>caption .fa-pull-right.headerlink,.rst-content tt.download span.fa-pull-right:first-child,.wy-menu-vertical li.current>a button.fa-pull-right.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-right.toctree-expand,.wy-menu-vertical li button.fa-pull-right.toctree-expand{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left,.pull-left.icon,.rst-content .code-block-caption .pull-left.headerlink,.rst-content .eqno .pull-left.headerlink,.rst-content .pull-left.admonition-title,.rst-content code.download span.pull-left:first-child,.rst-content dl dt .pull-left.headerlink,.rst-content h1 .pull-left.headerlink,.rst-content h2 .pull-left.headerlink,.rst-content h3 .pull-left.headerlink,.rst-content h4 .pull-left.headerlink,.rst-content h5 .pull-left.headerlink,.rst-content h6 .pull-left.headerlink,.rst-content p .pull-left.headerlink,.rst-content table>caption .pull-left.headerlink,.rst-content tt.download span.pull-left:first-child,.wy-menu-vertical li.current>a button.pull-left.toctree-expand,.wy-menu-vertical li.on a button.pull-left.toctree-expand,.wy-menu-vertical li button.pull-left.toctree-expand{margin-right:.3em}.fa.pull-right,.pull-right.icon,.rst-content .code-block-caption .pull-right.headerlink,.rst-content .eqno .pull-right.headerlink,.rst-content .pull-right.admonition-title,.rst-content code.download span.pull-right:first-child,.rst-content dl dt .pull-right.headerlink,.rst-content h1 .pull-right.headerlink,.rst-content h2 .pull-right.headerlink,.rst-content h3 .pull-right.headerlink,.rst-content h4 .pull-right.headerlink,.rst-content h5 .pull-right.headerlink,.rst-content h6 .pull-right.headerlink,.rst-content p .pull-right.headerlink,.rst-content table>caption .pull-right.headerlink,.rst-content tt.download span.pull-right:first-child,.wy-menu-vertical li.current>a button.pull-right.toctree-expand,.wy-menu-vertical li.on a button.pull-right.toctree-expand,.wy-menu-vertical li button.pull-right.toctree-expand{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s linear infinite;animation:fa-spin 2s linear infinite}.fa-pulse{-webkit-animation:fa-spin 1s steps(8) infinite;animation:fa-spin 1s steps(8) infinite}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scaleX(-1);-ms-transform:scaleX(-1);transform:scaleX(-1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scaleY(-1);-ms-transform:scaleY(-1);transform:scaleY(-1)}:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:""}.fa-music:before{content:""}.fa-search:before,.icon-search:before{content:""}.fa-envelope-o:before{content:""}.fa-heart:before{content:""}.fa-star:before{content:""}.fa-star-o:before{content:""}.fa-user:before{content:""}.fa-film:before{content:""}.fa-th-large:before{content:""}.fa-th:before{content:""}.fa-th-list:before{content:""}.fa-check:before{content:""}.fa-close:before,.fa-remove:before,.fa-times:before{content:""}.fa-search-plus:before{content:""}.fa-search-minus:before{content:""}.fa-power-off:before{content:""}.fa-signal:before{content:""}.fa-cog:before,.fa-gear:before{content:""}.fa-trash-o:before{content:""}.fa-home:before,.icon-home:before{content:""}.fa-file-o:before{content:""}.fa-clock-o:before{content:""}.fa-road:before{content:""}.fa-download:before,.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{content:""}.fa-arrow-circle-o-down:before{content:""}.fa-arrow-circle-o-up:before{content:""}.fa-inbox:before{content:""}.fa-play-circle-o:before{content:""}.fa-repeat:before,.fa-rotate-right:before{content:""}.fa-refresh:before{content:""}.fa-list-alt:before{content:""}.fa-lock:before{content:""}.fa-flag:before{content:""}.fa-headphones:before{content:""}.fa-volume-off:before{content:""}.fa-volume-down:before{content:""}.fa-volume-up:before{content:""}.fa-qrcode:before{content:""}.fa-barcode:before{content:""}.fa-tag:before{content:""}.fa-tags:before{content:""}.fa-book:before,.icon-book:before{content:""}.fa-bookmark:before{content:""}.fa-print:before{content:""}.fa-camera:before{content:""}.fa-font:before{content:""}.fa-bold:before{content:""}.fa-italic:before{content:""}.fa-text-height:before{content:""}.fa-text-width:before{content:""}.fa-align-left:before{content:""}.fa-align-center:before{content:""}.fa-align-right:before{content:""}.fa-align-justify:before{content:""}.fa-list:before{content:""}.fa-dedent:before,.fa-outdent:before{content:""}.fa-indent:before{content:""}.fa-video-camera:before{content:""}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:""}.fa-pencil:before{content:""}.fa-map-marker:before{content:""}.fa-adjust:before{content:""}.fa-tint:before{content:""}.fa-edit:before,.fa-pencil-square-o:before{content:""}.fa-share-square-o:before{content:""}.fa-check-square-o:before{content:""}.fa-arrows:before{content:""}.fa-step-backward:before{content:""}.fa-fast-backward:before{content:""}.fa-backward:before{content:""}.fa-play:before{content:""}.fa-pause:before{content:""}.fa-stop:before{content:""}.fa-forward:before{content:""}.fa-fast-forward:before{content:""}.fa-step-forward:before{content:""}.fa-eject:before{content:""}.fa-chevron-left:before{content:""}.fa-chevron-right:before{content:""}.fa-plus-circle:before{content:""}.fa-minus-circle:before{content:""}.fa-times-circle:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before{content:""}.fa-check-circle:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before{content:""}.fa-question-circle:before{content:""}.fa-info-circle:before{content:""}.fa-crosshairs:before{content:""}.fa-times-circle-o:before{content:""}.fa-check-circle-o:before{content:""}.fa-ban:before{content:""}.fa-arrow-left:before{content:""}.fa-arrow-right:before{content:""}.fa-arrow-up:before{content:""}.fa-arrow-down:before{content:""}.fa-mail-forward:before,.fa-share:before{content:""}.fa-expand:before{content:""}.fa-compress:before{content:""}.fa-plus:before{content:""}.fa-minus:before{content:""}.fa-asterisk:before{content:""}.fa-exclamation-circle:before,.rst-content .admonition-title:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before{content:""}.fa-gift:before{content:""}.fa-leaf:before{content:""}.fa-fire:before,.icon-fire:before{content:""}.fa-eye:before{content:""}.fa-eye-slash:before{content:""}.fa-exclamation-triangle:before,.fa-warning:before{content:""}.fa-plane:before{content:""}.fa-calendar:before{content:""}.fa-random:before{content:""}.fa-comment:before{content:""}.fa-magnet:before{content:""}.fa-chevron-up:before{content:""}.fa-chevron-down:before{content:""}.fa-retweet:before{content:""}.fa-shopping-cart:before{content:""}.fa-folder:before{content:""}.fa-folder-open:before{content:""}.fa-arrows-v:before{content:""}.fa-arrows-h:before{content:""}.fa-bar-chart-o:before,.fa-bar-chart:before{content:""}.fa-twitter-square:before{content:""}.fa-facebook-square:before{content:""}.fa-camera-retro:before{content:""}.fa-key:before{content:""}.fa-cogs:before,.fa-gears:before{content:""}.fa-comments:before{content:""}.fa-thumbs-o-up:before{content:""}.fa-thumbs-o-down:before{content:""}.fa-star-half:before{content:""}.fa-heart-o:before{content:""}.fa-sign-out:before{content:""}.fa-linkedin-square:before{content:""}.fa-thumb-tack:before{content:""}.fa-external-link:before{content:""}.fa-sign-in:before{content:""}.fa-trophy:before{content:""}.fa-github-square:before{content:""}.fa-upload:before{content:""}.fa-lemon-o:before{content:""}.fa-phone:before{content:""}.fa-square-o:before{content:""}.fa-bookmark-o:before{content:""}.fa-phone-square:before{content:""}.fa-twitter:before{content:""}.fa-facebook-f:before,.fa-facebook:before{content:""}.fa-github:before,.icon-github:before{content:""}.fa-unlock:before{content:""}.fa-credit-card:before{content:""}.fa-feed:before,.fa-rss:before{content:""}.fa-hdd-o:before{content:""}.fa-bullhorn:before{content:""}.fa-bell:before{content:""}.fa-certificate:before{content:""}.fa-hand-o-right:before{content:""}.fa-hand-o-left:before{content:""}.fa-hand-o-up:before{content:""}.fa-hand-o-down:before{content:""}.fa-arrow-circle-left:before,.icon-circle-arrow-left:before{content:""}.fa-arrow-circle-right:before,.icon-circle-arrow-right:before{content:""}.fa-arrow-circle-up:before{content:""}.fa-arrow-circle-down:before{content:""}.fa-globe:before{content:""}.fa-wrench:before{content:""}.fa-tasks:before{content:""}.fa-filter:before{content:""}.fa-briefcase:before{content:""}.fa-arrows-alt:before{content:""}.fa-group:before,.fa-users:before{content:""}.fa-chain:before,.fa-link:before,.icon-link:before{content:""}.fa-cloud:before{content:""}.fa-flask:before{content:""}.fa-cut:before,.fa-scissors:before{content:""}.fa-copy:before,.fa-files-o:before{content:""}.fa-paperclip:before{content:""}.fa-floppy-o:before,.fa-save:before{content:""}.fa-square:before{content:""}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:""}.fa-list-ul:before{content:""}.fa-list-ol:before{content:""}.fa-strikethrough:before{content:""}.fa-underline:before{content:""}.fa-table:before{content:""}.fa-magic:before{content:""}.fa-truck:before{content:""}.fa-pinterest:before{content:""}.fa-pinterest-square:before{content:""}.fa-google-plus-square:before{content:""}.fa-google-plus:before{content:""}.fa-money:before{content:""}.fa-caret-down:before,.icon-caret-down:before,.wy-dropdown .caret:before{content:""}.fa-caret-up:before{content:""}.fa-caret-left:before{content:""}.fa-caret-right:before{content:""}.fa-columns:before{content:""}.fa-sort:before,.fa-unsorted:before{content:""}.fa-sort-desc:before,.fa-sort-down:before{content:""}.fa-sort-asc:before,.fa-sort-up:before{content:""}.fa-envelope:before{content:""}.fa-linkedin:before{content:""}.fa-rotate-left:before,.fa-undo:before{content:""}.fa-gavel:before,.fa-legal:before{content:""}.fa-dashboard:before,.fa-tachometer:before{content:""}.fa-comment-o:before{content:""}.fa-comments-o:before{content:""}.fa-bolt:before,.fa-flash:before{content:""}.fa-sitemap:before{content:""}.fa-umbrella:before{content:""}.fa-clipboard:before,.fa-paste:before{content:""}.fa-lightbulb-o:before{content:""}.fa-exchange:before{content:""}.fa-cloud-download:before{content:""}.fa-cloud-upload:before{content:""}.fa-user-md:before{content:""}.fa-stethoscope:before{content:""}.fa-suitcase:before{content:""}.fa-bell-o:before{content:""}.fa-coffee:before{content:""}.fa-cutlery:before{content:""}.fa-file-text-o:before{content:""}.fa-building-o:before{content:""}.fa-hospital-o:before{content:""}.fa-ambulance:before{content:""}.fa-medkit:before{content:""}.fa-fighter-jet:before{content:""}.fa-beer:before{content:""}.fa-h-square:before{content:""}.fa-plus-square:before{content:""}.fa-angle-double-left:before{content:""}.fa-angle-double-right:before{content:""}.fa-angle-double-up:before{content:""}.fa-angle-double-down:before{content:""}.fa-angle-left:before{content:""}.fa-angle-right:before{content:""}.fa-angle-up:before{content:""}.fa-angle-down:before{content:""}.fa-desktop:before{content:""}.fa-laptop:before{content:""}.fa-tablet:before{content:""}.fa-mobile-phone:before,.fa-mobile:before{content:""}.fa-circle-o:before{content:""}.fa-quote-left:before{content:""}.fa-quote-right:before{content:""}.fa-spinner:before{content:""}.fa-circle:before{content:""}.fa-mail-reply:before,.fa-reply:before{content:""}.fa-github-alt:before{content:""}.fa-folder-o:before{content:""}.fa-folder-open-o:before{content:""}.fa-smile-o:before{content:""}.fa-frown-o:before{content:""}.fa-meh-o:before{content:""}.fa-gamepad:before{content:""}.fa-keyboard-o:before{content:""}.fa-flag-o:before{content:""}.fa-flag-checkered:before{content:""}.fa-terminal:before{content:""}.fa-code:before{content:""}.fa-mail-reply-all:before,.fa-reply-all:before{content:""}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:""}.fa-location-arrow:before{content:""}.fa-crop:before{content:""}.fa-code-fork:before{content:""}.fa-chain-broken:before,.fa-unlink:before{content:""}.fa-question:before{content:""}.fa-info:before{content:""}.fa-exclamation:before{content:""}.fa-superscript:before{content:""}.fa-subscript:before{content:""}.fa-eraser:before{content:""}.fa-puzzle-piece:before{content:""}.fa-microphone:before{content:""}.fa-microphone-slash:before{content:""}.fa-shield:before{content:""}.fa-calendar-o:before{content:""}.fa-fire-extinguisher:before{content:""}.fa-rocket:before{content:""}.fa-maxcdn:before{content:""}.fa-chevron-circle-left:before{content:""}.fa-chevron-circle-right:before{content:""}.fa-chevron-circle-up:before{content:""}.fa-chevron-circle-down:before{content:""}.fa-html5:before{content:""}.fa-css3:before{content:""}.fa-anchor:before{content:""}.fa-unlock-alt:before{content:""}.fa-bullseye:before{content:""}.fa-ellipsis-h:before{content:""}.fa-ellipsis-v:before{content:""}.fa-rss-square:before{content:""}.fa-play-circle:before{content:""}.fa-ticket:before{content:""}.fa-minus-square:before{content:""}.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before{content:""}.fa-level-up:before{content:""}.fa-level-down:before{content:""}.fa-check-square:before{content:""}.fa-pencil-square:before{content:""}.fa-external-link-square:before{content:""}.fa-share-square:before{content:""}.fa-compass:before{content:""}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:""}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:""}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:""}.fa-eur:before,.fa-euro:before{content:""}.fa-gbp:before{content:""}.fa-dollar:before,.fa-usd:before{content:""}.fa-inr:before,.fa-rupee:before{content:""}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:""}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:""}.fa-krw:before,.fa-won:before{content:""}.fa-bitcoin:before,.fa-btc:before{content:""}.fa-file:before{content:""}.fa-file-text:before{content:""}.fa-sort-alpha-asc:before{content:""}.fa-sort-alpha-desc:before{content:""}.fa-sort-amount-asc:before{content:""}.fa-sort-amount-desc:before{content:""}.fa-sort-numeric-asc:before{content:""}.fa-sort-numeric-desc:before{content:""}.fa-thumbs-up:before{content:""}.fa-thumbs-down:before{content:""}.fa-youtube-square:before{content:""}.fa-youtube:before{content:""}.fa-xing:before{content:""}.fa-xing-square:before{content:""}.fa-youtube-play:before{content:""}.fa-dropbox:before{content:""}.fa-stack-overflow:before{content:""}.fa-instagram:before{content:""}.fa-flickr:before{content:""}.fa-adn:before{content:""}.fa-bitbucket:before,.icon-bitbucket:before{content:""}.fa-bitbucket-square:before{content:""}.fa-tumblr:before{content:""}.fa-tumblr-square:before{content:""}.fa-long-arrow-down:before{content:""}.fa-long-arrow-up:before{content:""}.fa-long-arrow-left:before{content:""}.fa-long-arrow-right:before{content:""}.fa-apple:before{content:""}.fa-windows:before{content:""}.fa-android:before{content:""}.fa-linux:before{content:""}.fa-dribbble:before{content:""}.fa-skype:before{content:""}.fa-foursquare:before{content:""}.fa-trello:before{content:""}.fa-female:before{content:""}.fa-male:before{content:""}.fa-gittip:before,.fa-gratipay:before{content:""}.fa-sun-o:before{content:""}.fa-moon-o:before{content:""}.fa-archive:before{content:""}.fa-bug:before{content:""}.fa-vk:before{content:""}.fa-weibo:before{content:""}.fa-renren:before{content:""}.fa-pagelines:before{content:""}.fa-stack-exchange:before{content:""}.fa-arrow-circle-o-right:before{content:""}.fa-arrow-circle-o-left:before{content:""}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:""}.fa-dot-circle-o:before{content:""}.fa-wheelchair:before{content:""}.fa-vimeo-square:before{content:""}.fa-try:before,.fa-turkish-lira:before{content:""}.fa-plus-square-o:before,.wy-menu-vertical li button.toctree-expand:before{content:""}.fa-space-shuttle:before{content:""}.fa-slack:before{content:""}.fa-envelope-square:before{content:""}.fa-wordpress:before{content:""}.fa-openid:before{content:""}.fa-bank:before,.fa-institution:before,.fa-university:before{content:""}.fa-graduation-cap:before,.fa-mortar-board:before{content:""}.fa-yahoo:before{content:""}.fa-google:before{content:""}.fa-reddit:before{content:""}.fa-reddit-square:before{content:""}.fa-stumbleupon-circle:before{content:""}.fa-stumbleupon:before{content:""}.fa-delicious:before{content:""}.fa-digg:before{content:""}.fa-pied-piper-pp:before{content:""}.fa-pied-piper-alt:before{content:""}.fa-drupal:before{content:""}.fa-joomla:before{content:""}.fa-language:before{content:""}.fa-fax:before{content:""}.fa-building:before{content:""}.fa-child:before{content:""}.fa-paw:before{content:""}.fa-spoon:before{content:""}.fa-cube:before{content:""}.fa-cubes:before{content:""}.fa-behance:before{content:""}.fa-behance-square:before{content:""}.fa-steam:before{content:""}.fa-steam-square:before{content:""}.fa-recycle:before{content:""}.fa-automobile:before,.fa-car:before{content:""}.fa-cab:before,.fa-taxi:before{content:""}.fa-tree:before{content:""}.fa-spotify:before{content:""}.fa-deviantart:before{content:""}.fa-soundcloud:before{content:""}.fa-database:before{content:""}.fa-file-pdf-o:before{content:""}.fa-file-word-o:before{content:""}.fa-file-excel-o:before{content:""}.fa-file-powerpoint-o:before{content:""}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:""}.fa-file-archive-o:before,.fa-file-zip-o:before{content:""}.fa-file-audio-o:before,.fa-file-sound-o:before{content:""}.fa-file-movie-o:before,.fa-file-video-o:before{content:""}.fa-file-code-o:before{content:""}.fa-vine:before{content:""}.fa-codepen:before{content:""}.fa-jsfiddle:before{content:""}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:""}.fa-circle-o-notch:before{content:""}.fa-ra:before,.fa-rebel:before,.fa-resistance:before{content:""}.fa-empire:before,.fa-ge:before{content:""}.fa-git-square:before{content:""}.fa-git:before{content:""}.fa-hacker-news:before,.fa-y-combinator-square:before,.fa-yc-square:before{content:""}.fa-tencent-weibo:before{content:""}.fa-qq:before{content:""}.fa-wechat:before,.fa-weixin:before{content:""}.fa-paper-plane:before,.fa-send:before{content:""}.fa-paper-plane-o:before,.fa-send-o:before{content:""}.fa-history:before{content:""}.fa-circle-thin:before{content:""}.fa-header:before{content:""}.fa-paragraph:before{content:""}.fa-sliders:before{content:""}.fa-share-alt:before{content:""}.fa-share-alt-square:before{content:""}.fa-bomb:before{content:""}.fa-futbol-o:before,.fa-soccer-ball-o:before{content:""}.fa-tty:before{content:""}.fa-binoculars:before{content:""}.fa-plug:before{content:""}.fa-slideshare:before{content:""}.fa-twitch:before{content:""}.fa-yelp:before{content:""}.fa-newspaper-o:before{content:""}.fa-wifi:before{content:""}.fa-calculator:before{content:""}.fa-paypal:before{content:""}.fa-google-wallet:before{content:""}.fa-cc-visa:before{content:""}.fa-cc-mastercard:before{content:""}.fa-cc-discover:before{content:""}.fa-cc-amex:before{content:""}.fa-cc-paypal:before{content:""}.fa-cc-stripe:before{content:""}.fa-bell-slash:before{content:""}.fa-bell-slash-o:before{content:""}.fa-trash:before{content:""}.fa-copyright:before{content:""}.fa-at:before{content:""}.fa-eyedropper:before{content:""}.fa-paint-brush:before{content:""}.fa-birthday-cake:before{content:""}.fa-area-chart:before{content:""}.fa-pie-chart:before{content:""}.fa-line-chart:before{content:""}.fa-lastfm:before{content:""}.fa-lastfm-square:before{content:""}.fa-toggle-off:before{content:""}.fa-toggle-on:before{content:""}.fa-bicycle:before{content:""}.fa-bus:before{content:""}.fa-ioxhost:before{content:""}.fa-angellist:before{content:""}.fa-cc:before{content:""}.fa-ils:before,.fa-shekel:before,.fa-sheqel:before{content:""}.fa-meanpath:before{content:""}.fa-buysellads:before{content:""}.fa-connectdevelop:before{content:""}.fa-dashcube:before{content:""}.fa-forumbee:before{content:""}.fa-leanpub:before{content:""}.fa-sellsy:before{content:""}.fa-shirtsinbulk:before{content:""}.fa-simplybuilt:before{content:""}.fa-skyatlas:before{content:""}.fa-cart-plus:before{content:""}.fa-cart-arrow-down:before{content:""}.fa-diamond:before{content:""}.fa-ship:before{content:""}.fa-user-secret:before{content:""}.fa-motorcycle:before{content:""}.fa-street-view:before{content:""}.fa-heartbeat:before{content:""}.fa-venus:before{content:""}.fa-mars:before{content:""}.fa-mercury:before{content:""}.fa-intersex:before,.fa-transgender:before{content:""}.fa-transgender-alt:before{content:""}.fa-venus-double:before{content:""}.fa-mars-double:before{content:""}.fa-venus-mars:before{content:""}.fa-mars-stroke:before{content:""}.fa-mars-stroke-v:before{content:""}.fa-mars-stroke-h:before{content:""}.fa-neuter:before{content:""}.fa-genderless:before{content:""}.fa-facebook-official:before{content:""}.fa-pinterest-p:before{content:""}.fa-whatsapp:before{content:""}.fa-server:before{content:""}.fa-user-plus:before{content:""}.fa-user-times:before{content:""}.fa-bed:before,.fa-hotel:before{content:""}.fa-viacoin:before{content:""}.fa-train:before{content:""}.fa-subway:before{content:""}.fa-medium:before{content:""}.fa-y-combinator:before,.fa-yc:before{content:""}.fa-optin-monster:before{content:""}.fa-opencart:before{content:""}.fa-expeditedssl:before{content:""}.fa-battery-4:before,.fa-battery-full:before,.fa-battery:before{content:""}.fa-battery-3:before,.fa-battery-three-quarters:before{content:""}.fa-battery-2:before,.fa-battery-half:before{content:""}.fa-battery-1:before,.fa-battery-quarter:before{content:""}.fa-battery-0:before,.fa-battery-empty:before{content:""}.fa-mouse-pointer:before{content:""}.fa-i-cursor:before{content:""}.fa-object-group:before{content:""}.fa-object-ungroup:before{content:""}.fa-sticky-note:before{content:""}.fa-sticky-note-o:before{content:""}.fa-cc-jcb:before{content:""}.fa-cc-diners-club:before{content:""}.fa-clone:before{content:""}.fa-balance-scale:before{content:""}.fa-hourglass-o:before{content:""}.fa-hourglass-1:before,.fa-hourglass-start:before{content:""}.fa-hourglass-2:before,.fa-hourglass-half:before{content:""}.fa-hourglass-3:before,.fa-hourglass-end:before{content:""}.fa-hourglass:before{content:""}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:""}.fa-hand-paper-o:before,.fa-hand-stop-o:before{content:""}.fa-hand-scissors-o:before{content:""}.fa-hand-lizard-o:before{content:""}.fa-hand-spock-o:before{content:""}.fa-hand-pointer-o:before{content:""}.fa-hand-peace-o:before{content:""}.fa-trademark:before{content:""}.fa-registered:before{content:""}.fa-creative-commons:before{content:""}.fa-gg:before{content:""}.fa-gg-circle:before{content:""}.fa-tripadvisor:before{content:""}.fa-odnoklassniki:before{content:""}.fa-odnoklassniki-square:before{content:""}.fa-get-pocket:before{content:""}.fa-wikipedia-w:before{content:""}.fa-safari:before{content:""}.fa-chrome:before{content:""}.fa-firefox:before{content:""}.fa-opera:before{content:""}.fa-internet-explorer:before{content:""}.fa-television:before,.fa-tv:before{content:""}.fa-contao:before{content:""}.fa-500px:before{content:""}.fa-amazon:before{content:""}.fa-calendar-plus-o:before{content:""}.fa-calendar-minus-o:before{content:""}.fa-calendar-times-o:before{content:""}.fa-calendar-check-o:before{content:""}.fa-industry:before{content:""}.fa-map-pin:before{content:""}.fa-map-signs:before{content:""}.fa-map-o:before{content:""}.fa-map:before{content:""}.fa-commenting:before{content:""}.fa-commenting-o:before{content:""}.fa-houzz:before{content:""}.fa-vimeo:before{content:""}.fa-black-tie:before{content:""}.fa-fonticons:before{content:""}.fa-reddit-alien:before{content:""}.fa-edge:before{content:""}.fa-credit-card-alt:before{content:""}.fa-codiepie:before{content:""}.fa-modx:before{content:""}.fa-fort-awesome:before{content:""}.fa-usb:before{content:""}.fa-product-hunt:before{content:""}.fa-mixcloud:before{content:""}.fa-scribd:before{content:""}.fa-pause-circle:before{content:""}.fa-pause-circle-o:before{content:""}.fa-stop-circle:before{content:""}.fa-stop-circle-o:before{content:""}.fa-shopping-bag:before{content:""}.fa-shopping-basket:before{content:""}.fa-hashtag:before{content:""}.fa-bluetooth:before{content:""}.fa-bluetooth-b:before{content:""}.fa-percent:before{content:""}.fa-gitlab:before,.icon-gitlab:before{content:""}.fa-wpbeginner:before{content:""}.fa-wpforms:before{content:""}.fa-envira:before{content:""}.fa-universal-access:before{content:""}.fa-wheelchair-alt:before{content:""}.fa-question-circle-o:before{content:""}.fa-blind:before{content:""}.fa-audio-description:before{content:""}.fa-volume-control-phone:before{content:""}.fa-braille:before{content:""}.fa-assistive-listening-systems:before{content:""}.fa-american-sign-language-interpreting:before,.fa-asl-interpreting:before{content:""}.fa-deaf:before,.fa-deafness:before,.fa-hard-of-hearing:before{content:""}.fa-glide:before{content:""}.fa-glide-g:before{content:""}.fa-sign-language:before,.fa-signing:before{content:""}.fa-low-vision:before{content:""}.fa-viadeo:before{content:""}.fa-viadeo-square:before{content:""}.fa-snapchat:before{content:""}.fa-snapchat-ghost:before{content:""}.fa-snapchat-square:before{content:""}.fa-pied-piper:before{content:""}.fa-first-order:before{content:""}.fa-yoast:before{content:""}.fa-themeisle:before{content:""}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:""}.fa-fa:before,.fa-font-awesome:before{content:""}.fa-handshake-o:before{content:""}.fa-envelope-open:before{content:""}.fa-envelope-open-o:before{content:""}.fa-linode:before{content:""}.fa-address-book:before{content:""}.fa-address-book-o:before{content:""}.fa-address-card:before,.fa-vcard:before{content:""}.fa-address-card-o:before,.fa-vcard-o:before{content:""}.fa-user-circle:before{content:""}.fa-user-circle-o:before{content:""}.fa-user-o:before{content:""}.fa-id-badge:before{content:""}.fa-drivers-license:before,.fa-id-card:before{content:""}.fa-drivers-license-o:before,.fa-id-card-o:before{content:""}.fa-quora:before{content:""}.fa-free-code-camp:before{content:""}.fa-telegram:before{content:""}.fa-thermometer-4:before,.fa-thermometer-full:before,.fa-thermometer:before{content:""}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:""}.fa-thermometer-2:before,.fa-thermometer-half:before{content:""}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:""}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:""}.fa-shower:before{content:""}.fa-bath:before,.fa-bathtub:before,.fa-s15:before{content:""}.fa-podcast:before{content:""}.fa-window-maximize:before{content:""}.fa-window-minimize:before{content:""}.fa-window-restore:before{content:""}.fa-times-rectangle:before,.fa-window-close:before{content:""}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:""}.fa-bandcamp:before{content:""}.fa-grav:before{content:""}.fa-etsy:before{content:""}.fa-imdb:before{content:""}.fa-ravelry:before{content:""}.fa-eercast:before{content:""}.fa-microchip:before{content:""}.fa-snowflake-o:before{content:""}.fa-superpowers:before{content:""}.fa-wpexplorer:before{content:""}.fa-meetup:before{content:""}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-dropdown .caret,.wy-inline-validate.wy-inline-validate-danger .wy-input-context,.wy-inline-validate.wy-inline-validate-info .wy-input-context,.wy-inline-validate.wy-inline-validate-success .wy-input-context,.wy-inline-validate.wy-inline-validate-warning .wy-input-context,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{font-family:inherit}.fa:before,.icon:before,.rst-content .admonition-title:before,.rst-content .code-block-caption .headerlink:before,.rst-content .eqno .headerlink:before,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before{font-family:FontAwesome;display:inline-block;font-style:normal;font-weight:400;line-height:1;text-decoration:inherit}.rst-content .code-block-caption a .headerlink,.rst-content .eqno a .headerlink,.rst-content a .admonition-title,.rst-content code.download a span:first-child,.rst-content dl dt a .headerlink,.rst-content h1 a .headerlink,.rst-content h2 a .headerlink,.rst-content h3 a .headerlink,.rst-content h4 a .headerlink,.rst-content h5 a .headerlink,.rst-content h6 a .headerlink,.rst-content p.caption a .headerlink,.rst-content p a .headerlink,.rst-content table>caption a .headerlink,.rst-content tt.download a span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li a button.toctree-expand,a .fa,a .icon,a .rst-content .admonition-title,a .rst-content .code-block-caption .headerlink,a .rst-content .eqno .headerlink,a .rst-content code.download span:first-child,a .rst-content dl dt .headerlink,a .rst-content h1 .headerlink,a .rst-content h2 .headerlink,a .rst-content h3 .headerlink,a .rst-content h4 .headerlink,a .rst-content h5 .headerlink,a .rst-content h6 .headerlink,a .rst-content p.caption .headerlink,a .rst-content p .headerlink,a .rst-content table>caption .headerlink,a .rst-content tt.download span:first-child,a .wy-menu-vertical li button.toctree-expand{display:inline-block;text-decoration:inherit}.btn .fa,.btn .icon,.btn .rst-content .admonition-title,.btn .rst-content .code-block-caption .headerlink,.btn .rst-content .eqno .headerlink,.btn .rst-content code.download span:first-child,.btn .rst-content dl dt .headerlink,.btn .rst-content h1 .headerlink,.btn .rst-content h2 .headerlink,.btn .rst-content h3 .headerlink,.btn .rst-content h4 .headerlink,.btn .rst-content h5 .headerlink,.btn .rst-content h6 .headerlink,.btn .rst-content p .headerlink,.btn .rst-content table>caption .headerlink,.btn .rst-content tt.download span:first-child,.btn .wy-menu-vertical li.current>a button.toctree-expand,.btn .wy-menu-vertical li.on a button.toctree-expand,.btn .wy-menu-vertical li button.toctree-expand,.nav .fa,.nav .icon,.nav .rst-content .admonition-title,.nav .rst-content .code-block-caption .headerlink,.nav .rst-content .eqno .headerlink,.nav .rst-content code.download span:first-child,.nav .rst-content dl dt .headerlink,.nav .rst-content h1 .headerlink,.nav .rst-content h2 .headerlink,.nav .rst-content h3 .headerlink,.nav .rst-content h4 .headerlink,.nav .rst-content h5 .headerlink,.nav .rst-content h6 .headerlink,.nav .rst-content p .headerlink,.nav .rst-content table>caption .headerlink,.nav .rst-content tt.download span:first-child,.nav .wy-menu-vertical li.current>a button.toctree-expand,.nav .wy-menu-vertical li.on a button.toctree-expand,.nav .wy-menu-vertical li button.toctree-expand,.rst-content .btn .admonition-title,.rst-content .code-block-caption .btn .headerlink,.rst-content .code-block-caption .nav .headerlink,.rst-content .eqno .btn .headerlink,.rst-content .eqno .nav .headerlink,.rst-content .nav .admonition-title,.rst-content code.download .btn span:first-child,.rst-content code.download .nav span:first-child,.rst-content dl dt .btn .headerlink,.rst-content dl dt .nav .headerlink,.rst-content h1 .btn .headerlink,.rst-content h1 .nav .headerlink,.rst-content h2 .btn .headerlink,.rst-content h2 .nav .headerlink,.rst-content h3 .btn .headerlink,.rst-content h3 .nav .headerlink,.rst-content h4 .btn .headerlink,.rst-content h4 .nav .headerlink,.rst-content h5 .btn .headerlink,.rst-content h5 .nav .headerlink,.rst-content h6 .btn .headerlink,.rst-content h6 .nav .headerlink,.rst-content p .btn .headerlink,.rst-content p .nav .headerlink,.rst-content table>caption .btn .headerlink,.rst-content table>caption .nav .headerlink,.rst-content tt.download .btn span:first-child,.rst-content tt.download .nav span:first-child,.wy-menu-vertical li .btn button.toctree-expand,.wy-menu-vertical li.current>a .btn button.toctree-expand,.wy-menu-vertical li.current>a .nav button.toctree-expand,.wy-menu-vertical li .nav button.toctree-expand,.wy-menu-vertical li.on a .btn button.toctree-expand,.wy-menu-vertical li.on a .nav button.toctree-expand{display:inline}.btn .fa-large.icon,.btn .fa.fa-large,.btn .rst-content .code-block-caption .fa-large.headerlink,.btn .rst-content .eqno .fa-large.headerlink,.btn .rst-content .fa-large.admonition-title,.btn .rst-content code.download span.fa-large:first-child,.btn .rst-content dl dt .fa-large.headerlink,.btn .rst-content h1 .fa-large.headerlink,.btn .rst-content h2 .fa-large.headerlink,.btn .rst-content h3 .fa-large.headerlink,.btn .rst-content h4 .fa-large.headerlink,.btn .rst-content h5 .fa-large.headerlink,.btn .rst-content h6 .fa-large.headerlink,.btn .rst-content p .fa-large.headerlink,.btn .rst-content table>caption .fa-large.headerlink,.btn .rst-content tt.download span.fa-large:first-child,.btn .wy-menu-vertical li button.fa-large.toctree-expand,.nav .fa-large.icon,.nav .fa.fa-large,.nav .rst-content .code-block-caption .fa-large.headerlink,.nav .rst-content .eqno .fa-large.headerlink,.nav .rst-content .fa-large.admonition-title,.nav .rst-content code.download span.fa-large:first-child,.nav .rst-content dl dt .fa-large.headerlink,.nav .rst-content h1 .fa-large.headerlink,.nav .rst-content h2 .fa-large.headerlink,.nav .rst-content h3 .fa-large.headerlink,.nav .rst-content h4 .fa-large.headerlink,.nav .rst-content h5 .fa-large.headerlink,.nav .rst-content h6 .fa-large.headerlink,.nav .rst-content p .fa-large.headerlink,.nav .rst-content table>caption .fa-large.headerlink,.nav .rst-content tt.download span.fa-large:first-child,.nav .wy-menu-vertical li button.fa-large.toctree-expand,.rst-content .btn .fa-large.admonition-title,.rst-content .code-block-caption .btn .fa-large.headerlink,.rst-content .code-block-caption .nav .fa-large.headerlink,.rst-content .eqno .btn .fa-large.headerlink,.rst-content .eqno .nav .fa-large.headerlink,.rst-content .nav .fa-large.admonition-title,.rst-content code.download .btn span.fa-large:first-child,.rst-content code.download .nav span.fa-large:first-child,.rst-content dl dt .btn .fa-large.headerlink,.rst-content dl dt .nav .fa-large.headerlink,.rst-content h1 .btn .fa-large.headerlink,.rst-content h1 .nav .fa-large.headerlink,.rst-content h2 .btn .fa-large.headerlink,.rst-content h2 .nav .fa-large.headerlink,.rst-content h3 .btn .fa-large.headerlink,.rst-content h3 .nav .fa-large.headerlink,.rst-content h4 .btn .fa-large.headerlink,.rst-content h4 .nav .fa-large.headerlink,.rst-content h5 .btn .fa-large.headerlink,.rst-content h5 .nav .fa-large.headerlink,.rst-content h6 .btn .fa-large.headerlink,.rst-content h6 .nav .fa-large.headerlink,.rst-content p .btn .fa-large.headerlink,.rst-content p .nav .fa-large.headerlink,.rst-content table>caption .btn .fa-large.headerlink,.rst-content table>caption .nav .fa-large.headerlink,.rst-content tt.download .btn span.fa-large:first-child,.rst-content tt.download .nav span.fa-large:first-child,.wy-menu-vertical li .btn button.fa-large.toctree-expand,.wy-menu-vertical li .nav button.fa-large.toctree-expand{line-height:.9em}.btn .fa-spin.icon,.btn .fa.fa-spin,.btn .rst-content .code-block-caption .fa-spin.headerlink,.btn .rst-content .eqno .fa-spin.headerlink,.btn .rst-content .fa-spin.admonition-title,.btn .rst-content code.download span.fa-spin:first-child,.btn .rst-content dl dt .fa-spin.headerlink,.btn .rst-content h1 .fa-spin.headerlink,.btn .rst-content h2 .fa-spin.headerlink,.btn .rst-content h3 .fa-spin.headerlink,.btn .rst-content h4 .fa-spin.headerlink,.btn .rst-content h5 .fa-spin.headerlink,.btn .rst-content h6 .fa-spin.headerlink,.btn .rst-content p .fa-spin.headerlink,.btn .rst-content table>caption .fa-spin.headerlink,.btn .rst-content tt.download span.fa-spin:first-child,.btn .wy-menu-vertical li button.fa-spin.toctree-expand,.nav .fa-spin.icon,.nav .fa.fa-spin,.nav .rst-content .code-block-caption .fa-spin.headerlink,.nav .rst-content .eqno .fa-spin.headerlink,.nav .rst-content .fa-spin.admonition-title,.nav .rst-content code.download span.fa-spin:first-child,.nav .rst-content dl dt .fa-spin.headerlink,.nav .rst-content h1 .fa-spin.headerlink,.nav .rst-content h2 .fa-spin.headerlink,.nav .rst-content h3 .fa-spin.headerlink,.nav .rst-content h4 .fa-spin.headerlink,.nav .rst-content h5 .fa-spin.headerlink,.nav .rst-content h6 .fa-spin.headerlink,.nav .rst-content p .fa-spin.headerlink,.nav .rst-content table>caption .fa-spin.headerlink,.nav .rst-content tt.download span.fa-spin:first-child,.nav .wy-menu-vertical li button.fa-spin.toctree-expand,.rst-content .btn .fa-spin.admonition-title,.rst-content .code-block-caption .btn .fa-spin.headerlink,.rst-content .code-block-caption .nav .fa-spin.headerlink,.rst-content .eqno .btn .fa-spin.headerlink,.rst-content .eqno .nav .fa-spin.headerlink,.rst-content .nav .fa-spin.admonition-title,.rst-content code.download .btn span.fa-spin:first-child,.rst-content code.download .nav span.fa-spin:first-child,.rst-content dl dt .btn .fa-spin.headerlink,.rst-content dl dt .nav .fa-spin.headerlink,.rst-content h1 .btn .fa-spin.headerlink,.rst-content h1 .nav .fa-spin.headerlink,.rst-content h2 .btn .fa-spin.headerlink,.rst-content h2 .nav .fa-spin.headerlink,.rst-content h3 .btn .fa-spin.headerlink,.rst-content h3 .nav .fa-spin.headerlink,.rst-content h4 .btn .fa-spin.headerlink,.rst-content h4 .nav .fa-spin.headerlink,.rst-content h5 .btn .fa-spin.headerlink,.rst-content h5 .nav .fa-spin.headerlink,.rst-content h6 .btn .fa-spin.headerlink,.rst-content h6 .nav .fa-spin.headerlink,.rst-content p .btn .fa-spin.headerlink,.rst-content p .nav .fa-spin.headerlink,.rst-content table>caption .btn .fa-spin.headerlink,.rst-content table>caption .nav .fa-spin.headerlink,.rst-content tt.download .btn span.fa-spin:first-child,.rst-content tt.download .nav span.fa-spin:first-child,.wy-menu-vertical li .btn button.fa-spin.toctree-expand,.wy-menu-vertical li .nav button.fa-spin.toctree-expand{display:inline-block}.btn.fa:before,.btn.icon:before,.rst-content .btn.admonition-title:before,.rst-content .code-block-caption .btn.headerlink:before,.rst-content .eqno .btn.headerlink:before,.rst-content code.download span.btn:first-child:before,.rst-content dl dt .btn.headerlink:before,.rst-content h1 .btn.headerlink:before,.rst-content h2 .btn.headerlink:before,.rst-content h3 .btn.headerlink:before,.rst-content h4 .btn.headerlink:before,.rst-content h5 .btn.headerlink:before,.rst-content h6 .btn.headerlink:before,.rst-content p .btn.headerlink:before,.rst-content table>caption .btn.headerlink:before,.rst-content tt.download span.btn:first-child:before,.wy-menu-vertical li button.btn.toctree-expand:before{opacity:.5;-webkit-transition:opacity .05s ease-in;-moz-transition:opacity .05s ease-in;transition:opacity .05s ease-in}.btn.fa:hover:before,.btn.icon:hover:before,.rst-content .btn.admonition-title:hover:before,.rst-content .code-block-caption .btn.headerlink:hover:before,.rst-content .eqno .btn.headerlink:hover:before,.rst-content code.download span.btn:first-child:hover:before,.rst-content dl dt .btn.headerlink:hover:before,.rst-content h1 .btn.headerlink:hover:before,.rst-content h2 .btn.headerlink:hover:before,.rst-content h3 .btn.headerlink:hover:before,.rst-content h4 .btn.headerlink:hover:before,.rst-content h5 .btn.headerlink:hover:before,.rst-content h6 .btn.headerlink:hover:before,.rst-content p .btn.headerlink:hover:before,.rst-content table>caption .btn.headerlink:hover:before,.rst-content tt.download span.btn:first-child:hover:before,.wy-menu-vertical li button.btn.toctree-expand:hover:before{opacity:1}.btn-mini .fa:before,.btn-mini .icon:before,.btn-mini .rst-content .admonition-title:before,.btn-mini .rst-content .code-block-caption .headerlink:before,.btn-mini .rst-content .eqno .headerlink:before,.btn-mini .rst-content code.download span:first-child:before,.btn-mini .rst-content dl dt .headerlink:before,.btn-mini .rst-content h1 .headerlink:before,.btn-mini .rst-content h2 .headerlink:before,.btn-mini .rst-content h3 .headerlink:before,.btn-mini .rst-content h4 .headerlink:before,.btn-mini .rst-content h5 .headerlink:before,.btn-mini .rst-content h6 .headerlink:before,.btn-mini .rst-content p .headerlink:before,.btn-mini .rst-content table>caption .headerlink:before,.btn-mini .rst-content tt.download span:first-child:before,.btn-mini .wy-menu-vertical li button.toctree-expand:before,.rst-content .btn-mini .admonition-title:before,.rst-content .code-block-caption .btn-mini .headerlink:before,.rst-content .eqno .btn-mini .headerlink:before,.rst-content code.download .btn-mini span:first-child:before,.rst-content dl dt .btn-mini .headerlink:before,.rst-content h1 .btn-mini .headerlink:before,.rst-content h2 .btn-mini .headerlink:before,.rst-content h3 .btn-mini .headerlink:before,.rst-content h4 .btn-mini .headerlink:before,.rst-content h5 .btn-mini .headerlink:before,.rst-content h6 .btn-mini .headerlink:before,.rst-content p .btn-mini .headerlink:before,.rst-content table>caption .btn-mini .headerlink:before,.rst-content tt.download .btn-mini span:first-child:before,.wy-menu-vertical li .btn-mini button.toctree-expand:before{font-size:14px;vertical-align:-15%}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.wy-alert{padding:12px;line-height:24px;margin-bottom:24px;background:#e7f2fa}.rst-content .admonition-title,.wy-alert-title{font-weight:700;display:block;color:#fff;background:#6ab0de;padding:6px 12px;margin:-12px -12px 12px}.rst-content .danger,.rst-content .error,.rst-content .wy-alert-danger.admonition,.rst-content .wy-alert-danger.admonition-todo,.rst-content .wy-alert-danger.attention,.rst-content .wy-alert-danger.caution,.rst-content .wy-alert-danger.hint,.rst-content .wy-alert-danger.important,.rst-content .wy-alert-danger.note,.rst-content .wy-alert-danger.seealso,.rst-content .wy-alert-danger.tip,.rst-content .wy-alert-danger.warning,.wy-alert.wy-alert-danger{background:#fdf3f2}.rst-content .danger .admonition-title,.rst-content .danger .wy-alert-title,.rst-content .error .admonition-title,.rst-content .error .wy-alert-title,.rst-content .wy-alert-danger.admonition-todo .admonition-title,.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,.rst-content .wy-alert-danger.admonition .admonition-title,.rst-content .wy-alert-danger.admonition .wy-alert-title,.rst-content .wy-alert-danger.attention .admonition-title,.rst-content .wy-alert-danger.attention .wy-alert-title,.rst-content .wy-alert-danger.caution .admonition-title,.rst-content .wy-alert-danger.caution .wy-alert-title,.rst-content .wy-alert-danger.hint .admonition-title,.rst-content .wy-alert-danger.hint .wy-alert-title,.rst-content .wy-alert-danger.important .admonition-title,.rst-content .wy-alert-danger.important .wy-alert-title,.rst-content .wy-alert-danger.note .admonition-title,.rst-content .wy-alert-danger.note .wy-alert-title,.rst-content .wy-alert-danger.seealso .admonition-title,.rst-content .wy-alert-danger.seealso .wy-alert-title,.rst-content .wy-alert-danger.tip .admonition-title,.rst-content .wy-alert-danger.tip .wy-alert-title,.rst-content .wy-alert-danger.warning .admonition-title,.rst-content .wy-alert-danger.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-danger .admonition-title,.wy-alert.wy-alert-danger .rst-content .admonition-title,.wy-alert.wy-alert-danger .wy-alert-title{background:#f29f97}.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .warning,.rst-content .wy-alert-warning.admonition,.rst-content .wy-alert-warning.danger,.rst-content .wy-alert-warning.error,.rst-content .wy-alert-warning.hint,.rst-content .wy-alert-warning.important,.rst-content .wy-alert-warning.note,.rst-content .wy-alert-warning.seealso,.rst-content .wy-alert-warning.tip,.wy-alert.wy-alert-warning{background:#ffedcc}.rst-content .admonition-todo .admonition-title,.rst-content .admonition-todo .wy-alert-title,.rst-content .attention .admonition-title,.rst-content .attention .wy-alert-title,.rst-content .caution .admonition-title,.rst-content .caution .wy-alert-title,.rst-content .warning .admonition-title,.rst-content .warning .wy-alert-title,.rst-content .wy-alert-warning.admonition .admonition-title,.rst-content .wy-alert-warning.admonition .wy-alert-title,.rst-content .wy-alert-warning.danger .admonition-title,.rst-content .wy-alert-warning.danger .wy-alert-title,.rst-content .wy-alert-warning.error .admonition-title,.rst-content .wy-alert-warning.error .wy-alert-title,.rst-content .wy-alert-warning.hint .admonition-title,.rst-content .wy-alert-warning.hint .wy-alert-title,.rst-content .wy-alert-warning.important .admonition-title,.rst-content .wy-alert-warning.important .wy-alert-title,.rst-content .wy-alert-warning.note .admonition-title,.rst-content .wy-alert-warning.note .wy-alert-title,.rst-content .wy-alert-warning.seealso .admonition-title,.rst-content .wy-alert-warning.seealso .wy-alert-title,.rst-content .wy-alert-warning.tip .admonition-title,.rst-content .wy-alert-warning.tip .wy-alert-title,.rst-content .wy-alert.wy-alert-warning .admonition-title,.wy-alert.wy-alert-warning .rst-content .admonition-title,.wy-alert.wy-alert-warning .wy-alert-title{background:#f0b37e}.rst-content .note,.rst-content .seealso,.rst-content .wy-alert-info.admonition,.rst-content .wy-alert-info.admonition-todo,.rst-content .wy-alert-info.attention,.rst-content .wy-alert-info.caution,.rst-content .wy-alert-info.danger,.rst-content .wy-alert-info.error,.rst-content .wy-alert-info.hint,.rst-content .wy-alert-info.important,.rst-content .wy-alert-info.tip,.rst-content .wy-alert-info.warning,.wy-alert.wy-alert-info{background:#e7f2fa}.rst-content .note .admonition-title,.rst-content .note .wy-alert-title,.rst-content .seealso .admonition-title,.rst-content .seealso .wy-alert-title,.rst-content .wy-alert-info.admonition-todo .admonition-title,.rst-content .wy-alert-info.admonition-todo .wy-alert-title,.rst-content .wy-alert-info.admonition .admonition-title,.rst-content .wy-alert-info.admonition .wy-alert-title,.rst-content .wy-alert-info.attention .admonition-title,.rst-content .wy-alert-info.attention .wy-alert-title,.rst-content .wy-alert-info.caution .admonition-title,.rst-content .wy-alert-info.caution .wy-alert-title,.rst-content .wy-alert-info.danger .admonition-title,.rst-content .wy-alert-info.danger .wy-alert-title,.rst-content .wy-alert-info.error .admonition-title,.rst-content .wy-alert-info.error .wy-alert-title,.rst-content .wy-alert-info.hint .admonition-title,.rst-content .wy-alert-info.hint .wy-alert-title,.rst-content .wy-alert-info.important .admonition-title,.rst-content .wy-alert-info.important .wy-alert-title,.rst-content .wy-alert-info.tip .admonition-title,.rst-content .wy-alert-info.tip .wy-alert-title,.rst-content .wy-alert-info.warning .admonition-title,.rst-content .wy-alert-info.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-info .admonition-title,.wy-alert.wy-alert-info .rst-content .admonition-title,.wy-alert.wy-alert-info .wy-alert-title{background:#6ab0de}.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .wy-alert-success.admonition,.rst-content .wy-alert-success.admonition-todo,.rst-content .wy-alert-success.attention,.rst-content .wy-alert-success.caution,.rst-content .wy-alert-success.danger,.rst-content .wy-alert-success.error,.rst-content .wy-alert-success.note,.rst-content .wy-alert-success.seealso,.rst-content .wy-alert-success.warning,.wy-alert.wy-alert-success{background:#dbfaf4}.rst-content .hint .admonition-title,.rst-content .hint .wy-alert-title,.rst-content .important .admonition-title,.rst-content .important .wy-alert-title,.rst-content .tip .admonition-title,.rst-content .tip .wy-alert-title,.rst-content .wy-alert-success.admonition-todo .admonition-title,.rst-content .wy-alert-success.admonition-todo .wy-alert-title,.rst-content .wy-alert-success.admonition .admonition-title,.rst-content .wy-alert-success.admonition .wy-alert-title,.rst-content .wy-alert-success.attention .admonition-title,.rst-content .wy-alert-success.attention .wy-alert-title,.rst-content .wy-alert-success.caution .admonition-title,.rst-content .wy-alert-success.caution .wy-alert-title,.rst-content .wy-alert-success.danger .admonition-title,.rst-content .wy-alert-success.danger .wy-alert-title,.rst-content .wy-alert-success.error .admonition-title,.rst-content .wy-alert-success.error .wy-alert-title,.rst-content .wy-alert-success.note .admonition-title,.rst-content .wy-alert-success.note .wy-alert-title,.rst-content .wy-alert-success.seealso .admonition-title,.rst-content .wy-alert-success.seealso .wy-alert-title,.rst-content .wy-alert-success.warning .admonition-title,.rst-content .wy-alert-success.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-success .admonition-title,.wy-alert.wy-alert-success .rst-content .admonition-title,.wy-alert.wy-alert-success .wy-alert-title{background:#1abc9c}.rst-content .wy-alert-neutral.admonition,.rst-content .wy-alert-neutral.admonition-todo,.rst-content .wy-alert-neutral.attention,.rst-content .wy-alert-neutral.caution,.rst-content .wy-alert-neutral.danger,.rst-content .wy-alert-neutral.error,.rst-content .wy-alert-neutral.hint,.rst-content .wy-alert-neutral.important,.rst-content .wy-alert-neutral.note,.rst-content .wy-alert-neutral.seealso,.rst-content .wy-alert-neutral.tip,.rst-content .wy-alert-neutral.warning,.wy-alert.wy-alert-neutral{background:#f3f6f6}.rst-content .wy-alert-neutral.admonition-todo .admonition-title,.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,.rst-content .wy-alert-neutral.admonition .admonition-title,.rst-content .wy-alert-neutral.admonition .wy-alert-title,.rst-content .wy-alert-neutral.attention .admonition-title,.rst-content .wy-alert-neutral.attention .wy-alert-title,.rst-content .wy-alert-neutral.caution .admonition-title,.rst-content .wy-alert-neutral.caution .wy-alert-title,.rst-content .wy-alert-neutral.danger .admonition-title,.rst-content .wy-alert-neutral.danger .wy-alert-title,.rst-content .wy-alert-neutral.error .admonition-title,.rst-content .wy-alert-neutral.error .wy-alert-title,.rst-content .wy-alert-neutral.hint .admonition-title,.rst-content .wy-alert-neutral.hint .wy-alert-title,.rst-content .wy-alert-neutral.important .admonition-title,.rst-content .wy-alert-neutral.important .wy-alert-title,.rst-content .wy-alert-neutral.note .admonition-title,.rst-content .wy-alert-neutral.note .wy-alert-title,.rst-content .wy-alert-neutral.seealso .admonition-title,.rst-content .wy-alert-neutral.seealso .wy-alert-title,.rst-content .wy-alert-neutral.tip .admonition-title,.rst-content .wy-alert-neutral.tip .wy-alert-title,.rst-content .wy-alert-neutral.warning .admonition-title,.rst-content .wy-alert-neutral.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-neutral .admonition-title,.wy-alert.wy-alert-neutral .rst-content .admonition-title,.wy-alert.wy-alert-neutral .wy-alert-title{color:#404040;background:#e1e4e5}.rst-content .wy-alert-neutral.admonition-todo a,.rst-content .wy-alert-neutral.admonition a,.rst-content .wy-alert-neutral.attention a,.rst-content .wy-alert-neutral.caution a,.rst-content .wy-alert-neutral.danger a,.rst-content .wy-alert-neutral.error a,.rst-content .wy-alert-neutral.hint a,.rst-content .wy-alert-neutral.important a,.rst-content .wy-alert-neutral.note a,.rst-content .wy-alert-neutral.seealso a,.rst-content .wy-alert-neutral.tip a,.rst-content .wy-alert-neutral.warning a,.wy-alert.wy-alert-neutral a{color:#2980b9}.rst-content .admonition-todo p:last-child,.rst-content .admonition p:last-child,.rst-content .attention p:last-child,.rst-content .caution p:last-child,.rst-content .danger p:last-child,.rst-content .error p:last-child,.rst-content .hint p:last-child,.rst-content .important p:last-child,.rst-content .note p:last-child,.rst-content .seealso p:last-child,.rst-content .tip p:last-child,.rst-content .warning p:last-child,.wy-alert p:last-child{margin-bottom:0}.wy-tray-container{position:fixed;bottom:0;left:0;z-index:600}.wy-tray-container li{display:block;width:300px;background:transparent;color:#fff;text-align:center;box-shadow:0 5px 5px 0 rgba(0,0,0,.1);padding:0 24px;min-width:20%;opacity:0;height:0;line-height:56px;overflow:hidden;-webkit-transition:all .3s ease-in;-moz-transition:all .3s ease-in;transition:all .3s ease-in}.wy-tray-container li.wy-tray-item-success{background:#27ae60}.wy-tray-container li.wy-tray-item-info{background:#2980b9}.wy-tray-container li.wy-tray-item-warning{background:#e67e22}.wy-tray-container li.wy-tray-item-danger{background:#e74c3c}.wy-tray-container li.on{opacity:1;height:56px}@media screen and (max-width:768px){.wy-tray-container{bottom:auto;top:0;width:100%}.wy-tray-container li{width:100%}}button{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle;cursor:pointer;line-height:normal;-webkit-appearance:button;*overflow:visible}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}button[disabled]{cursor:default}.btn{display:inline-block;border-radius:2px;line-height:normal;white-space:nowrap;text-align:center;cursor:pointer;font-size:100%;padding:6px 12px 8px;color:#fff;border:1px solid rgba(0,0,0,.1);background-color:#27ae60;text-decoration:none;font-weight:400;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 2px -1px hsla(0,0%,100%,.5),inset 0 -2px 0 0 rgba(0,0,0,.1);outline-none:false;vertical-align:middle;*display:inline;zoom:1;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:all .1s linear;-moz-transition:all .1s linear;transition:all .1s linear}.btn-hover{background:#2e8ece;color:#fff}.btn:hover{background:#2cc36b;color:#fff}.btn:focus{background:#2cc36b;outline:0}.btn:active{box-shadow:inset 0 -1px 0 0 rgba(0,0,0,.05),inset 0 2px 0 0 rgba(0,0,0,.1);padding:8px 12px 6px}.btn:visited{color:#fff}.btn-disabled,.btn-disabled:active,.btn-disabled:focus,.btn-disabled:hover,.btn:disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:.4;cursor:not-allowed;box-shadow:none}.btn::-moz-focus-inner{padding:0;border:0}.btn-small{font-size:80%}.btn-info{background-color:#2980b9!important}.btn-info:hover{background-color:#2e8ece!important}.btn-neutral{background-color:#f3f6f6!important;color:#404040!important}.btn-neutral:hover{background-color:#e5ebeb!important;color:#404040}.btn-neutral:visited{color:#404040!important}.btn-success{background-color:#27ae60!important}.btn-success:hover{background-color:#295!important}.btn-danger{background-color:#e74c3c!important}.btn-danger:hover{background-color:#ea6153!important}.btn-warning{background-color:#e67e22!important}.btn-warning:hover{background-color:#e98b39!important}.btn-invert{background-color:#222}.btn-invert:hover{background-color:#2f2f2f!important}.btn-link{background-color:transparent!important;color:#2980b9;box-shadow:none;border-color:transparent!important}.btn-link:active,.btn-link:hover{background-color:transparent!important;color:#409ad5!important;box-shadow:none}.btn-link:visited{color:#9b59b6}.wy-btn-group .btn,.wy-control .btn{vertical-align:middle}.wy-btn-group{margin-bottom:24px;*zoom:1}.wy-btn-group:after,.wy-btn-group:before{display:table;content:""}.wy-btn-group:after{clear:both}.wy-dropdown{position:relative;display:inline-block}.wy-dropdown-active .wy-dropdown-menu{display:block}.wy-dropdown-menu{position:absolute;left:0;display:none;float:left;top:100%;min-width:100%;background:#fcfcfc;z-index:100;border:1px solid #cfd7dd;box-shadow:0 2px 2px 0 rgba(0,0,0,.1);padding:12px}.wy-dropdown-menu>dd>a{display:block;clear:both;color:#404040;white-space:nowrap;font-size:90%;padding:0 12px;cursor:pointer}.wy-dropdown-menu>dd>a:hover{background:#2980b9;color:#fff}.wy-dropdown-menu>dd.divider{border-top:1px solid #cfd7dd;margin:6px 0}.wy-dropdown-menu>dd.search{padding-bottom:12px}.wy-dropdown-menu>dd.search input[type=search]{width:100%}.wy-dropdown-menu>dd.call-to-action{background:#e3e3e3;text-transform:uppercase;font-weight:500;font-size:80%}.wy-dropdown-menu>dd.call-to-action:hover{background:#e3e3e3}.wy-dropdown-menu>dd.call-to-action .btn{color:#fff}.wy-dropdown.wy-dropdown-up .wy-dropdown-menu{bottom:100%;top:auto;left:auto;right:0}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu{background:#fcfcfc;margin-top:2px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a{padding:6px 12px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover{background:#2980b9;color:#fff}.wy-dropdown.wy-dropdown-left .wy-dropdown-menu{right:0;left:auto;text-align:right}.wy-dropdown-arrow:before{content:" ";border-bottom:5px solid #f5f5f5;border-left:5px solid transparent;border-right:5px solid transparent;position:absolute;display:block;top:-4px;left:50%;margin-left:-3px}.wy-dropdown-arrow.wy-dropdown-arrow-left:before{left:11px}.wy-form-stacked select{display:block}.wy-form-aligned .wy-help-inline,.wy-form-aligned input,.wy-form-aligned label,.wy-form-aligned select,.wy-form-aligned textarea{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-form-aligned .wy-control-group>label{display:inline-block;vertical-align:middle;width:10em;margin:6px 12px 0 0;float:left}.wy-form-aligned .wy-control{float:left}.wy-form-aligned .wy-control label{display:block}.wy-form-aligned .wy-control select{margin-top:6px}fieldset{margin:0}fieldset,legend{border:0;padding:0}legend{width:100%;white-space:normal;margin-bottom:24px;font-size:150%;*margin-left:-7px}label,legend{display:block}label{margin:0 0 .3125em;color:#333;font-size:90%}input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}.wy-control-group{margin-bottom:24px;max-width:1200px;margin-left:auto;margin-right:auto;*zoom:1}.wy-control-group:after,.wy-control-group:before{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group.wy-control-group-required>label:after{content:" *";color:#e74c3c}.wy-control-group .wy-form-full,.wy-control-group .wy-form-halves,.wy-control-group .wy-form-thirds{padding-bottom:12px}.wy-control-group .wy-form-full input[type=color],.wy-control-group .wy-form-full input[type=date],.wy-control-group .wy-form-full input[type=datetime-local],.wy-control-group .wy-form-full input[type=datetime],.wy-control-group .wy-form-full input[type=email],.wy-control-group .wy-form-full input[type=month],.wy-control-group .wy-form-full input[type=number],.wy-control-group .wy-form-full input[type=password],.wy-control-group .wy-form-full input[type=search],.wy-control-group .wy-form-full input[type=tel],.wy-control-group .wy-form-full input[type=text],.wy-control-group .wy-form-full input[type=time],.wy-control-group .wy-form-full input[type=url],.wy-control-group .wy-form-full input[type=week],.wy-control-group .wy-form-full select,.wy-control-group .wy-form-halves input[type=color],.wy-control-group .wy-form-halves input[type=date],.wy-control-group .wy-form-halves input[type=datetime-local],.wy-control-group .wy-form-halves input[type=datetime],.wy-control-group .wy-form-halves input[type=email],.wy-control-group .wy-form-halves input[type=month],.wy-control-group .wy-form-halves input[type=number],.wy-control-group .wy-form-halves input[type=password],.wy-control-group .wy-form-halves input[type=search],.wy-control-group .wy-form-halves input[type=tel],.wy-control-group .wy-form-halves input[type=text],.wy-control-group .wy-form-halves input[type=time],.wy-control-group .wy-form-halves input[type=url],.wy-control-group .wy-form-halves input[type=week],.wy-control-group .wy-form-halves select,.wy-control-group .wy-form-thirds input[type=color],.wy-control-group .wy-form-thirds input[type=date],.wy-control-group .wy-form-thirds input[type=datetime-local],.wy-control-group .wy-form-thirds input[type=datetime],.wy-control-group .wy-form-thirds input[type=email],.wy-control-group .wy-form-thirds input[type=month],.wy-control-group .wy-form-thirds input[type=number],.wy-control-group .wy-form-thirds input[type=password],.wy-control-group .wy-form-thirds input[type=search],.wy-control-group .wy-form-thirds input[type=tel],.wy-control-group .wy-form-thirds input[type=text],.wy-control-group .wy-form-thirds input[type=time],.wy-control-group .wy-form-thirds input[type=url],.wy-control-group .wy-form-thirds input[type=week],.wy-control-group .wy-form-thirds select{width:100%}.wy-control-group .wy-form-full{float:left;display:block;width:100%;margin-right:0}.wy-control-group .wy-form-full:last-child{margin-right:0}.wy-control-group .wy-form-halves{float:left;display:block;margin-right:2.35765%;width:48.82117%}.wy-control-group .wy-form-halves:last-child,.wy-control-group .wy-form-halves:nth-of-type(2n){margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(odd){clear:left}.wy-control-group .wy-form-thirds{float:left;display:block;margin-right:2.35765%;width:31.76157%}.wy-control-group .wy-form-thirds:last-child,.wy-control-group .wy-form-thirds:nth-of-type(3n){margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n+1){clear:left}.wy-control-group.wy-control-group-no-input .wy-control,.wy-control-no-input{margin:6px 0 0;font-size:90%}.wy-control-no-input{display:inline-block}.wy-control-group.fluid-input input[type=color],.wy-control-group.fluid-input input[type=date],.wy-control-group.fluid-input input[type=datetime-local],.wy-control-group.fluid-input input[type=datetime],.wy-control-group.fluid-input input[type=email],.wy-control-group.fluid-input input[type=month],.wy-control-group.fluid-input input[type=number],.wy-control-group.fluid-input input[type=password],.wy-control-group.fluid-input input[type=search],.wy-control-group.fluid-input input[type=tel],.wy-control-group.fluid-input input[type=text],.wy-control-group.fluid-input input[type=time],.wy-control-group.fluid-input input[type=url],.wy-control-group.fluid-input input[type=week]{width:100%}.wy-form-message-inline{padding-left:.3em;color:#666;font-size:90%}.wy-form-message{display:block;color:#999;font-size:70%;margin-top:.3125em;font-style:italic}.wy-form-message p{font-size:inherit;font-style:italic;margin-bottom:6px}.wy-form-message p:last-child{margin-bottom:0}input{line-height:normal}input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;*overflow:visible}input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week]{-webkit-appearance:none;padding:6px;display:inline-block;border:1px solid #ccc;font-size:80%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 3px #ddd;border-radius:0;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}input[type=datetime-local]{padding:.34375em .625em}input[disabled]{cursor:default}input[type=checkbox],input[type=radio]{padding:0;margin-right:.3125em;*height:13px;*width:13px}input[type=checkbox],input[type=radio],input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}input[type=color]:focus,input[type=date]:focus,input[type=datetime-local]:focus,input[type=datetime]:focus,input[type=email]:focus,input[type=month]:focus,input[type=number]:focus,input[type=password]:focus,input[type=search]:focus,input[type=tel]:focus,input[type=text]:focus,input[type=time]:focus,input[type=url]:focus,input[type=week]:focus{outline:0;outline:thin dotted\9;border-color:#333}input.no-focus:focus{border-color:#ccc!important}input[type=checkbox]:focus,input[type=file]:focus,input[type=radio]:focus{outline:thin dotted #333;outline:1px auto #129fea}input[type=color][disabled],input[type=date][disabled],input[type=datetime-local][disabled],input[type=datetime][disabled],input[type=email][disabled],input[type=month][disabled],input[type=number][disabled],input[type=password][disabled],input[type=search][disabled],input[type=tel][disabled],input[type=text][disabled],input[type=time][disabled],input[type=url][disabled],input[type=week][disabled]{cursor:not-allowed;background-color:#fafafa}input:focus:invalid,select:focus:invalid,textarea:focus:invalid{color:#e74c3c;border:1px solid #e74c3c}input:focus:invalid:focus,select:focus:invalid:focus,textarea:focus:invalid:focus{border-color:#e74c3c}input[type=checkbox]:focus:invalid:focus,input[type=file]:focus:invalid:focus,input[type=radio]:focus:invalid:focus{outline-color:#e74c3c}input.wy-input-large{padding:12px;font-size:100%}textarea{overflow:auto;vertical-align:top;width:100%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif}select,textarea{padding:.5em .625em;display:inline-block;border:1px solid #ccc;font-size:80%;box-shadow:inset 0 1px 3px #ddd;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}select{border:1px solid #ccc;background-color:#fff}select[multiple]{height:auto}select:focus,textarea:focus{outline:0}input[readonly],select[disabled],select[readonly],textarea[disabled],textarea[readonly]{cursor:not-allowed;background-color:#fafafa}input[type=checkbox][disabled],input[type=radio][disabled]{cursor:not-allowed}.wy-checkbox,.wy-radio{margin:6px 0;color:#404040;display:block}.wy-checkbox input,.wy-radio input{vertical-align:baseline}.wy-form-message-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-input-prefix,.wy-input-suffix{white-space:nowrap;padding:6px}.wy-input-prefix .wy-input-context,.wy-input-suffix .wy-input-context{line-height:27px;padding:0 8px;display:inline-block;font-size:80%;background-color:#f3f6f6;border:1px solid #ccc;color:#999}.wy-input-suffix .wy-input-context{border-left:0}.wy-input-prefix .wy-input-context{border-right:0}.wy-switch{position:relative;display:block;height:24px;margin-top:12px;cursor:pointer}.wy-switch:before{left:0;top:0;width:36px;height:12px;background:#ccc}.wy-switch:after,.wy-switch:before{position:absolute;content:"";display:block;border-radius:4px;-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.wy-switch:after{width:18px;height:18px;background:#999;left:-3px;top:-3px}.wy-switch span{position:absolute;left:48px;display:block;font-size:12px;color:#ccc;line-height:1}.wy-switch.active:before{background:#1e8449}.wy-switch.active:after{left:24px;background:#27ae60}.wy-switch.disabled{cursor:not-allowed;opacity:.8}.wy-control-group.wy-control-group-error .wy-form-message,.wy-control-group.wy-control-group-error>label{color:#e74c3c}.wy-control-group.wy-control-group-error input[type=color],.wy-control-group.wy-control-group-error input[type=date],.wy-control-group.wy-control-group-error input[type=datetime-local],.wy-control-group.wy-control-group-error input[type=datetime],.wy-control-group.wy-control-group-error input[type=email],.wy-control-group.wy-control-group-error input[type=month],.wy-control-group.wy-control-group-error input[type=number],.wy-control-group.wy-control-group-error input[type=password],.wy-control-group.wy-control-group-error input[type=search],.wy-control-group.wy-control-group-error input[type=tel],.wy-control-group.wy-control-group-error input[type=text],.wy-control-group.wy-control-group-error input[type=time],.wy-control-group.wy-control-group-error input[type=url],.wy-control-group.wy-control-group-error input[type=week],.wy-control-group.wy-control-group-error textarea{border:1px solid #e74c3c}.wy-inline-validate{white-space:nowrap}.wy-inline-validate .wy-input-context{padding:.5em .625em;display:inline-block;font-size:80%}.wy-inline-validate.wy-inline-validate-success .wy-input-context{color:#27ae60}.wy-inline-validate.wy-inline-validate-danger .wy-input-context{color:#e74c3c}.wy-inline-validate.wy-inline-validate-warning .wy-input-context{color:#e67e22}.wy-inline-validate.wy-inline-validate-info .wy-input-context{color:#2980b9}.rotate-90{-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.rotate-180{-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.rotate-270{-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.mirror{-webkit-transform:scaleX(-1);-moz-transform:scaleX(-1);-ms-transform:scaleX(-1);-o-transform:scaleX(-1);transform:scaleX(-1)}.mirror.rotate-90{-webkit-transform:scaleX(-1) rotate(90deg);-moz-transform:scaleX(-1) rotate(90deg);-ms-transform:scaleX(-1) rotate(90deg);-o-transform:scaleX(-1) rotate(90deg);transform:scaleX(-1) rotate(90deg)}.mirror.rotate-180{-webkit-transform:scaleX(-1) rotate(180deg);-moz-transform:scaleX(-1) rotate(180deg);-ms-transform:scaleX(-1) rotate(180deg);-o-transform:scaleX(-1) rotate(180deg);transform:scaleX(-1) rotate(180deg)}.mirror.rotate-270{-webkit-transform:scaleX(-1) rotate(270deg);-moz-transform:scaleX(-1) rotate(270deg);-ms-transform:scaleX(-1) rotate(270deg);-o-transform:scaleX(-1) rotate(270deg);transform:scaleX(-1) rotate(270deg)}@media only screen and (max-width:480px){.wy-form button[type=submit]{margin:.7em 0 0}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=text],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week],.wy-form label{margin-bottom:.3em;display:block}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week]{margin-bottom:0}.wy-form-aligned .wy-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.wy-form-aligned .wy-control{margin:1.5em 0 0}.wy-form-message,.wy-form-message-inline,.wy-form .wy-help-inline{display:block;font-size:80%;padding:6px 0}}@media screen and (max-width:768px){.tablet-hide{display:none}}@media screen and (max-width:480px){.mobile-hide{display:none}}.float-left{float:left}.float-right{float:right}.full-width{width:100%}.rst-content table.docutils,.rst-content table.field-list,.wy-table{border-collapse:collapse;border-spacing:0;empty-cells:show;margin-bottom:24px}.rst-content table.docutils caption,.rst-content table.field-list caption,.wy-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.rst-content table.docutils td,.rst-content table.docutils th,.rst-content table.field-list td,.rst-content table.field-list th,.wy-table td,.wy-table th{font-size:90%;margin:0;overflow:visible;padding:8px 16px}.rst-content table.docutils td:first-child,.rst-content table.docutils th:first-child,.rst-content table.field-list td:first-child,.rst-content table.field-list th:first-child,.wy-table td:first-child,.wy-table th:first-child{border-left-width:0}.rst-content table.docutils thead,.rst-content table.field-list thead,.wy-table thead{color:#000;text-align:left;vertical-align:bottom;white-space:nowrap}.rst-content table.docutils thead th,.rst-content table.field-list thead th,.wy-table thead th{font-weight:700;border-bottom:2px solid #e1e4e5}.rst-content table.docutils td,.rst-content table.field-list td,.wy-table td{background-color:transparent;vertical-align:middle}.rst-content table.docutils td p,.rst-content table.field-list td p,.wy-table td p{line-height:18px}.rst-content table.docutils td p:last-child,.rst-content table.field-list td p:last-child,.wy-table td p:last-child{margin-bottom:0}.rst-content table.docutils .wy-table-cell-min,.rst-content table.field-list .wy-table-cell-min,.wy-table .wy-table-cell-min{width:1%;padding-right:0}.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox],.wy-table .wy-table-cell-min input[type=checkbox]{margin:0}.wy-table-secondary{color:grey;font-size:90%}.wy-table-tertiary{color:grey;font-size:80%}.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,.wy-table-backed,.wy-table-odd td,.wy-table-striped tr:nth-child(2n-1) td{background-color:#f3f6f6}.rst-content table.docutils,.wy-table-bordered-all{border:1px solid #e1e4e5}.rst-content table.docutils td,.wy-table-bordered-all td{border-bottom:1px solid #e1e4e5;border-left:1px solid #e1e4e5}.rst-content table.docutils tbody>tr:last-child td,.wy-table-bordered-all tbody>tr:last-child td{border-bottom-width:0}.wy-table-bordered{border:1px solid #e1e4e5}.wy-table-bordered-rows td{border-bottom:1px solid #e1e4e5}.wy-table-bordered-rows tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal td,.wy-table-horizontal th{border-width:0 0 1px;border-bottom:1px solid #e1e4e5}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-responsive{margin-bottom:24px;max-width:100%;overflow:auto}.wy-table-responsive table{margin-bottom:0!important}.wy-table-responsive table td,.wy-table-responsive table th{white-space:nowrap}a{color:#2980b9;text-decoration:none;cursor:pointer}a:hover{color:#3091d1}a:visited{color:#9b59b6}html{height:100%}body,html{overflow-x:hidden}body{font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-weight:400;color:#404040;min-height:100%;background:#edf0f2}.wy-text-left{text-align:left}.wy-text-center{text-align:center}.wy-text-right{text-align:right}.wy-text-large{font-size:120%}.wy-text-normal{font-size:100%}.wy-text-small,small{font-size:80%}.wy-text-strike{text-decoration:line-through}.wy-text-warning{color:#e67e22!important}a.wy-text-warning:hover{color:#eb9950!important}.wy-text-info{color:#2980b9!important}a.wy-text-info:hover{color:#409ad5!important}.wy-text-success{color:#27ae60!important}a.wy-text-success:hover{color:#36d278!important}.wy-text-danger{color:#e74c3c!important}a.wy-text-danger:hover{color:#ed7669!important}.wy-text-neutral{color:#404040!important}a.wy-text-neutral:hover{color:#595959!important}.rst-content .toctree-wrapper>p.caption,h1,h2,h3,h4,h5,h6,legend{margin-top:0;font-weight:700;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif}p{line-height:24px;font-size:16px;margin:0 0 24px}h1{font-size:175%}.rst-content .toctree-wrapper>p.caption,h2{font-size:150%}h3{font-size:125%}h4{font-size:115%}h5{font-size:110%}h6{font-size:100%}hr{display:block;height:1px;border:0;border-top:1px solid #e1e4e5;margin:24px 0;padding:0}.rst-content code,.rst-content tt,code{white-space:nowrap;max-width:100%;background:#fff;border:1px solid #e1e4e5;font-size:75%;padding:0 5px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#e74c3c;overflow-x:auto}.rst-content tt.code-large,code.code-large{font-size:90%}.rst-content .section ul,.rst-content .toctree-wrapper ul,.rst-content section ul,.wy-plain-list-disc,article ul{list-style:disc;line-height:24px;margin-bottom:24px}.rst-content .section ul li,.rst-content .toctree-wrapper ul li,.rst-content section ul li,.wy-plain-list-disc li,article ul li{list-style:disc;margin-left:24px}.rst-content .section ul li p:last-child,.rst-content .section ul li ul,.rst-content .toctree-wrapper ul li p:last-child,.rst-content .toctree-wrapper ul li ul,.rst-content section ul li p:last-child,.rst-content section ul li ul,.wy-plain-list-disc li p:last-child,.wy-plain-list-disc li ul,article ul li p:last-child,article ul li ul{margin-bottom:0}.rst-content .section ul li li,.rst-content .toctree-wrapper ul li li,.rst-content section ul li li,.wy-plain-list-disc li li,article ul li li{list-style:circle}.rst-content .section ul li li li,.rst-content .toctree-wrapper ul li li li,.rst-content section ul li li li,.wy-plain-list-disc li li li,article ul li li li{list-style:square}.rst-content .section ul li ol li,.rst-content .toctree-wrapper ul li ol li,.rst-content section ul li ol li,.wy-plain-list-disc li ol li,article ul li ol li{list-style:decimal}.rst-content .section ol,.rst-content .section ol.arabic,.rst-content .toctree-wrapper ol,.rst-content .toctree-wrapper ol.arabic,.rst-content section ol,.rst-content section ol.arabic,.wy-plain-list-decimal,article ol{list-style:decimal;line-height:24px;margin-bottom:24px}.rst-content .section ol.arabic li,.rst-content .section ol li,.rst-content .toctree-wrapper ol.arabic li,.rst-content .toctree-wrapper ol li,.rst-content section ol.arabic li,.rst-content section ol li,.wy-plain-list-decimal li,article ol li{list-style:decimal;margin-left:24px}.rst-content .section ol.arabic li ul,.rst-content .section ol li p:last-child,.rst-content .section ol li ul,.rst-content .toctree-wrapper ol.arabic li ul,.rst-content .toctree-wrapper ol li p:last-child,.rst-content .toctree-wrapper ol li ul,.rst-content section ol.arabic li ul,.rst-content section ol li p:last-child,.rst-content section ol li ul,.wy-plain-list-decimal li p:last-child,.wy-plain-list-decimal li ul,article ol li p:last-child,article ol li ul{margin-bottom:0}.rst-content .section ol.arabic li ul li,.rst-content .section ol li ul li,.rst-content .toctree-wrapper ol.arabic li ul li,.rst-content .toctree-wrapper ol li ul li,.rst-content section ol.arabic li ul li,.rst-content section ol li ul li,.wy-plain-list-decimal li ul li,article ol li ul li{list-style:disc}.wy-breadcrumbs{*zoom:1}.wy-breadcrumbs:after,.wy-breadcrumbs:before{display:table;content:""}.wy-breadcrumbs:after{clear:both}.wy-breadcrumbs>li{display:inline-block;padding-top:5px}.wy-breadcrumbs>li.wy-breadcrumbs-aside{float:right}.rst-content .wy-breadcrumbs>li code,.rst-content .wy-breadcrumbs>li tt,.wy-breadcrumbs>li .rst-content tt,.wy-breadcrumbs>li code{all:inherit;color:inherit}.breadcrumb-item:before{content:"/";color:#bbb;font-size:13px;padding:0 6px 0 3px}.wy-breadcrumbs-extra{margin-bottom:0;color:#b3b3b3;font-size:80%;display:inline-block}@media screen and (max-width:480px){.wy-breadcrumbs-extra,.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}@media print{.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}html{font-size:16px}.wy-affix{position:fixed;top:1.618em}.wy-menu a:hover{text-decoration:none}.wy-menu-horiz{*zoom:1}.wy-menu-horiz:after,.wy-menu-horiz:before{display:table;content:""}.wy-menu-horiz:after{clear:both}.wy-menu-horiz li,.wy-menu-horiz ul{display:inline-block}.wy-menu-horiz li:hover{background:hsla(0,0%,100%,.1)}.wy-menu-horiz li.divide-left{border-left:1px solid #404040}.wy-menu-horiz li.divide-right{border-right:1px solid #404040}.wy-menu-horiz a{height:32px;display:inline-block;line-height:32px;padding:0 16px}.wy-menu-vertical{width:300px}.wy-menu-vertical header,.wy-menu-vertical p.caption{color:#55a5d9;height:32px;line-height:32px;padding:0 1.618em;margin:12px 0 0;display:block;font-weight:700;text-transform:uppercase;font-size:85%;white-space:nowrap}.wy-menu-vertical ul{margin-bottom:0}.wy-menu-vertical li.divide-top{border-top:1px solid #404040}.wy-menu-vertical li.divide-bottom{border-bottom:1px solid #404040}.wy-menu-vertical li.current{background:#e3e3e3}.wy-menu-vertical li.current a{color:grey;border-right:1px solid #c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.current a:hover{background:#d6d6d6}.rst-content .wy-menu-vertical li tt,.wy-menu-vertical li .rst-content tt,.wy-menu-vertical li code{border:none;background:inherit;color:inherit;padding-left:0;padding-right:0}.wy-menu-vertical li button.toctree-expand{display:block;float:left;margin-left:-1.2em;line-height:18px;color:#4d4d4d;border:none;background:none;padding:0}.wy-menu-vertical li.current>a,.wy-menu-vertical li.on a{color:#404040;font-weight:700;position:relative;background:#fcfcfc;border:none;padding:.4045em 1.618em}.wy-menu-vertical li.current>a:hover,.wy-menu-vertical li.on a:hover{background:#fcfcfc}.wy-menu-vertical li.current>a:hover button.toctree-expand,.wy-menu-vertical li.on a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand{display:block;line-height:18px;color:#333}.wy-menu-vertical li.toctree-l1.current>a{border-bottom:1px solid #c9c9c9;border-top:1px solid #c9c9c9}.wy-menu-vertical .toctree-l1.current .toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .toctree-l11>ul{display:none}.wy-menu-vertical .toctree-l1.current .current.toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .current.toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .current.toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .current.toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .current.toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .current.toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .current.toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .current.toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .current.toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .current.toctree-l11>ul{display:block}.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4{font-size:.9em}.wy-menu-vertical li.toctree-l2 a,.wy-menu-vertical li.toctree-l3 a,.wy-menu-vertical li.toctree-l4 a,.wy-menu-vertical li.toctree-l5 a,.wy-menu-vertical li.toctree-l6 a,.wy-menu-vertical li.toctree-l7 a,.wy-menu-vertical li.toctree-l8 a,.wy-menu-vertical li.toctree-l9 a,.wy-menu-vertical li.toctree-l10 a{color:#404040}.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l4 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l5 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l6 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l7 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l8 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l9 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l10 a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a,.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a,.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a,.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a,.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a,.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a,.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a,.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{display:block}.wy-menu-vertical li.toctree-l2.current>a{padding:.4045em 2.427em}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{padding:.4045em 1.618em .4045em 4.045em}.wy-menu-vertical li.toctree-l3.current>a{padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{padding:.4045em 1.618em .4045em 5.663em}.wy-menu-vertical li.toctree-l4.current>a{padding:.4045em 5.663em}.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a{padding:.4045em 1.618em .4045em 7.281em}.wy-menu-vertical li.toctree-l5.current>a{padding:.4045em 7.281em}.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a{padding:.4045em 1.618em .4045em 8.899em}.wy-menu-vertical li.toctree-l6.current>a{padding:.4045em 8.899em}.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a{padding:.4045em 1.618em .4045em 10.517em}.wy-menu-vertical li.toctree-l7.current>a{padding:.4045em 10.517em}.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a{padding:.4045em 1.618em .4045em 12.135em}.wy-menu-vertical li.toctree-l8.current>a{padding:.4045em 12.135em}.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a{padding:.4045em 1.618em .4045em 13.753em}.wy-menu-vertical li.toctree-l9.current>a{padding:.4045em 13.753em}.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a{padding:.4045em 1.618em .4045em 15.371em}.wy-menu-vertical li.toctree-l10.current>a{padding:.4045em 15.371em}.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{padding:.4045em 1.618em .4045em 16.989em}.wy-menu-vertical li.toctree-l2.current>a,.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{background:#c9c9c9}.wy-menu-vertical li.toctree-l2 button.toctree-expand{color:#a3a3a3}.wy-menu-vertical li.toctree-l3.current>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{background:#bdbdbd}.wy-menu-vertical li.toctree-l3 button.toctree-expand{color:#969696}.wy-menu-vertical li.current ul{display:block}.wy-menu-vertical li ul{margin-bottom:0;display:none}.wy-menu-vertical li ul li a{margin-bottom:0;color:#d9d9d9;font-weight:400}.wy-menu-vertical a{line-height:18px;padding:.4045em 1.618em;display:block;position:relative;font-size:90%;color:#d9d9d9}.wy-menu-vertical a:hover{background-color:#4e4a4a;cursor:pointer}.wy-menu-vertical a:hover button.toctree-expand{color:#d9d9d9}.wy-menu-vertical a:active{background-color:#2980b9;cursor:pointer;color:#fff}.wy-menu-vertical a:active button.toctree-expand{color:#fff}.wy-side-nav-search{display:block;width:300px;padding:.809em;margin-bottom:.809em;z-index:200;background-color:#2980b9;text-align:center;color:#fcfcfc}.wy-side-nav-search input[type=text]{width:100%;border-radius:50px;padding:6px 12px;border-color:#2472a4}.wy-side-nav-search img{display:block;margin:auto auto .809em;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-side-nav-search .wy-dropdown>a,.wy-side-nav-search>a{color:#fcfcfc;font-size:100%;font-weight:700;display:inline-block;padding:4px 6px;margin-bottom:.809em;max-width:100%}.wy-side-nav-search .wy-dropdown>a:hover,.wy-side-nav-search>a:hover{background:hsla(0,0%,100%,.1)}.wy-side-nav-search .wy-dropdown>a img.logo,.wy-side-nav-search>a img.logo{display:block;margin:0 auto;height:auto;width:auto;border-radius:0;max-width:100%;background:transparent}.wy-side-nav-search .wy-dropdown>a.icon img.logo,.wy-side-nav-search>a.icon img.logo{margin-top:.85em}.wy-side-nav-search>div.version{margin-top:-.4045em;margin-bottom:.809em;font-weight:400;color:hsla(0,0%,100%,.3)}.wy-nav .wy-menu-vertical header{color:#2980b9}.wy-nav .wy-menu-vertical a{color:#b3b3b3}.wy-nav .wy-menu-vertical a:hover{background-color:#2980b9;color:#fff}[data-menu-wrap]{-webkit-transition:all .2s ease-in;-moz-transition:all .2s ease-in;transition:all .2s ease-in;position:absolute;opacity:1;width:100%;opacity:0}[data-menu-wrap].move-center{left:0;right:auto;opacity:1}[data-menu-wrap].move-left{right:auto;left:-100%;opacity:0}[data-menu-wrap].move-right{right:-100%;left:auto;opacity:0}.wy-body-for-nav{background:#fcfcfc}.wy-grid-for-nav{position:absolute;width:100%;height:100%}.wy-nav-side{position:fixed;top:0;bottom:0;left:0;padding-bottom:2em;width:300px;overflow-x:hidden;overflow-y:hidden;min-height:100%;color:#9b9b9b;background:#343131;z-index:200}.wy-side-scroll{width:320px;position:relative;overflow-x:hidden;overflow-y:scroll;height:100%}.wy-nav-top{display:none;background:#2980b9;color:#fff;padding:.4045em .809em;position:relative;line-height:50px;text-align:center;font-size:100%;*zoom:1}.wy-nav-top:after,.wy-nav-top:before{display:table;content:""}.wy-nav-top:after{clear:both}.wy-nav-top a{color:#fff;font-weight:700}.wy-nav-top img{margin-right:12px;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-nav-top i{font-size:30px;float:left;cursor:pointer;padding-top:inherit}.wy-nav-content-wrap{margin-left:300px;background:#fcfcfc;min-height:100%}.wy-nav-content{padding:1.618em 3.236em;height:100%;max-width:800px;margin:auto}.wy-body-mask{position:fixed;width:100%;height:100%;background:rgba(0,0,0,.2);display:none;z-index:499}.wy-body-mask.on{display:block}footer{color:grey}footer p{margin-bottom:12px}.rst-content footer span.commit tt,footer span.commit .rst-content tt,footer span.commit code{padding:0;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:1em;background:none;border:none;color:grey}.rst-footer-buttons{*zoom:1}.rst-footer-buttons:after,.rst-footer-buttons:before{width:100%;display:table;content:""}.rst-footer-buttons:after{clear:both}.rst-breadcrumbs-buttons{margin-top:12px;*zoom:1}.rst-breadcrumbs-buttons:after,.rst-breadcrumbs-buttons:before{display:table;content:""}.rst-breadcrumbs-buttons:after{clear:both}#search-results .search li{margin-bottom:24px;border-bottom:1px solid #e1e4e5;padding-bottom:24px}#search-results .search li:first-child{border-top:1px solid #e1e4e5;padding-top:24px}#search-results .search li a{font-size:120%;margin-bottom:12px;display:inline-block}#search-results .context{color:grey;font-size:90%}.genindextable li>ul{margin-left:24px}@media screen and (max-width:768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}.wy-nav-side{left:-300px}.wy-nav-side.shift{width:85%;left:0}.wy-menu.wy-menu-vertical,.wy-side-nav-search,.wy-side-scroll{width:auto}.wy-nav-content-wrap{margin-left:0}.wy-nav-content-wrap .wy-nav-content{padding:1.618em}.wy-nav-content-wrap.shift{position:fixed;min-width:100%;left:85%;top:0;height:100%;overflow:hidden}}@media screen and (min-width:1100px){.wy-nav-content-wrap{background:rgba(0,0,0,.05)}.wy-nav-content{margin:0;background:#fcfcfc}}@media print{.rst-versions,.wy-nav-side,footer{display:none}.wy-nav-content-wrap{margin-left:0}}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60;*zoom:1}.rst-versions .rst-current-version:after,.rst-versions .rst-current-version:before{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-content .code-block-caption .rst-versions .rst-current-version .headerlink,.rst-content .eqno .rst-versions .rst-current-version .headerlink,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-content code.download .rst-versions .rst-current-version span:first-child,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-content p .rst-versions .rst-current-version .headerlink,.rst-content table>caption .rst-versions .rst-current-version .headerlink,.rst-content tt.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .icon,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-versions .rst-current-version .rst-content .code-block-caption .headerlink,.rst-versions .rst-current-version .rst-content .eqno .headerlink,.rst-versions .rst-current-version .rst-content code.download span:first-child,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-versions .rst-current-version .rst-content p .headerlink,.rst-versions .rst-current-version .rst-content table>caption .headerlink,.rst-versions .rst-current-version .rst-content tt.download span:first-child,.rst-versions .rst-current-version .wy-menu-vertical li button.toctree-expand,.wy-menu-vertical li .rst-versions .rst-current-version button.toctree-expand{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}.rst-content .toctree-wrapper>p.caption,.rst-content h1,.rst-content h2,.rst-content h3,.rst-content h4,.rst-content h5,.rst-content h6{margin-bottom:24px}.rst-content img{max-width:100%;height:auto}.rst-content div.figure,.rst-content figure{margin-bottom:24px}.rst-content div.figure .caption-text,.rst-content figure .caption-text{font-style:italic}.rst-content div.figure p:last-child.caption,.rst-content figure p:last-child.caption{margin-bottom:0}.rst-content div.figure.align-center,.rst-content figure.align-center{text-align:center}.rst-content .section>a>img,.rst-content .section>img,.rst-content section>a>img,.rst-content section>img{margin-bottom:24px}.rst-content abbr[title]{text-decoration:none}.rst-content.style-external-links a.reference.external:after{font-family:FontAwesome;content:"\f08e";color:#b3b3b3;vertical-align:super;font-size:60%;margin:0 .2em}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content pre.literal-block{white-space:pre;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;display:block;overflow:auto}.rst-content div[class^=highlight],.rst-content pre.literal-block{border:1px solid #e1e4e5;overflow-x:auto;margin:1px 0 24px}.rst-content div[class^=highlight] div[class^=highlight],.rst-content pre.literal-block div[class^=highlight]{padding:0;border:none;margin:0}.rst-content div[class^=highlight] td.code{width:100%}.rst-content .linenodiv pre{border-right:1px solid #e6e9ea;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;user-select:none;pointer-events:none}.rst-content div[class^=highlight] pre{white-space:pre;margin:0;padding:12px;display:block;overflow:auto}.rst-content div[class^=highlight] pre .hll{display:block;margin:0 -12px;padding:0 12px}.rst-content .linenodiv pre,.rst-content div[class^=highlight] pre,.rst-content pre.literal-block{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:12px;line-height:1.4}.rst-content div.highlight .gp,.rst-content div.highlight span.linenos{user-select:none;pointer-events:none}.rst-content div.highlight span.linenos{display:inline-block;padding-left:0;padding-right:12px;margin-right:12px;border-right:1px solid #e6e9ea}.rst-content .code-block-caption{font-style:italic;font-size:85%;line-height:1;padding:1em 0;text-align:center}@media print{.rst-content .codeblock,.rst-content div[class^=highlight],.rst-content div[class^=highlight] pre{white-space:pre-wrap}}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning{clear:both}.rst-content .admonition-todo .last,.rst-content .admonition-todo>:last-child,.rst-content .admonition .last,.rst-content .admonition>:last-child,.rst-content .attention .last,.rst-content .attention>:last-child,.rst-content .caution .last,.rst-content .caution>:last-child,.rst-content .danger .last,.rst-content .danger>:last-child,.rst-content .error .last,.rst-content .error>:last-child,.rst-content .hint .last,.rst-content .hint>:last-child,.rst-content .important .last,.rst-content .important>:last-child,.rst-content .note .last,.rst-content .note>:last-child,.rst-content .seealso .last,.rst-content .seealso>:last-child,.rst-content .tip .last,.rst-content .tip>:last-child,.rst-content .warning .last,.rst-content .warning>:last-child{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent!important;border-color:rgba(0,0,0,.1)!important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha>li,.rst-content .toctree-wrapper ol.loweralpha,.rst-content .toctree-wrapper ol.loweralpha>li,.rst-content section ol.loweralpha,.rst-content section ol.loweralpha>li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha>li,.rst-content .toctree-wrapper ol.upperalpha,.rst-content .toctree-wrapper ol.upperalpha>li,.rst-content section ol.upperalpha,.rst-content section ol.upperalpha>li{list-style:upper-alpha}.rst-content .section ol li>*,.rst-content .section ul li>*,.rst-content .toctree-wrapper ol li>*,.rst-content .toctree-wrapper ul li>*,.rst-content section ol li>*,.rst-content section ul li>*{margin-top:12px;margin-bottom:12px}.rst-content .section ol li>:first-child,.rst-content .section ul li>:first-child,.rst-content .toctree-wrapper ol li>:first-child,.rst-content .toctree-wrapper ul li>:first-child,.rst-content section ol li>:first-child,.rst-content section ul li>:first-child{margin-top:0}.rst-content .section ol li>p,.rst-content .section ol li>p:last-child,.rst-content .section ul li>p,.rst-content .section ul li>p:last-child,.rst-content .toctree-wrapper ol li>p,.rst-content .toctree-wrapper ol li>p:last-child,.rst-content .toctree-wrapper ul li>p,.rst-content .toctree-wrapper ul li>p:last-child,.rst-content section ol li>p,.rst-content section ol li>p:last-child,.rst-content section ul li>p,.rst-content section ul li>p:last-child{margin-bottom:12px}.rst-content .section ol li>p:only-child,.rst-content .section ol li>p:only-child:last-child,.rst-content .section ul li>p:only-child,.rst-content .section ul li>p:only-child:last-child,.rst-content .toctree-wrapper ol li>p:only-child,.rst-content .toctree-wrapper ol li>p:only-child:last-child,.rst-content .toctree-wrapper ul li>p:only-child,.rst-content .toctree-wrapper ul li>p:only-child:last-child,.rst-content section ol li>p:only-child,.rst-content section ol li>p:only-child:last-child,.rst-content section ul li>p:only-child,.rst-content section ul li>p:only-child:last-child{margin-bottom:0}.rst-content .section ol li>ol,.rst-content .section ol li>ul,.rst-content .section ul li>ol,.rst-content .section ul li>ul,.rst-content .toctree-wrapper ol li>ol,.rst-content .toctree-wrapper ol li>ul,.rst-content .toctree-wrapper ul li>ol,.rst-content .toctree-wrapper ul li>ul,.rst-content section ol li>ol,.rst-content section ol li>ul,.rst-content section ul li>ol,.rst-content section ul li>ul{margin-bottom:12px}.rst-content .section ol.simple li>*,.rst-content .section ol.simple li ol,.rst-content .section ol.simple li ul,.rst-content .section ul.simple li>*,.rst-content .section ul.simple li ol,.rst-content .section ul.simple li ul,.rst-content .toctree-wrapper ol.simple li>*,.rst-content .toctree-wrapper ol.simple li ol,.rst-content .toctree-wrapper ol.simple li ul,.rst-content .toctree-wrapper ul.simple li>*,.rst-content .toctree-wrapper ul.simple li ol,.rst-content .toctree-wrapper ul.simple li ul,.rst-content section ol.simple li>*,.rst-content section ol.simple li ol,.rst-content section ol.simple li ul,.rst-content section ul.simple li>*,.rst-content section ul.simple li ol,.rst-content section ul.simple li ul{margin-top:0;margin-bottom:0}.rst-content .line-block{margin-left:0;margin-bottom:24px;line-height:24px}.rst-content .line-block .line-block{margin-left:24px;margin-bottom:0}.rst-content .topic-title{font-weight:700;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0 0 24px 24px}.rst-content .align-left{float:left;margin:0 24px 24px 0}.rst-content .align-center{margin:auto}.rst-content .align-center:not(table){display:block}.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink{opacity:0;font-size:14px;font-family:FontAwesome;margin-left:.5em}.rst-content .code-block-caption .headerlink:focus,.rst-content .code-block-caption:hover .headerlink,.rst-content .eqno .headerlink:focus,.rst-content .eqno:hover .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink:focus,.rst-content .toctree-wrapper>p.caption:hover .headerlink,.rst-content dl dt .headerlink:focus,.rst-content dl dt:hover .headerlink,.rst-content h1 .headerlink:focus,.rst-content h1:hover .headerlink,.rst-content h2 .headerlink:focus,.rst-content h2:hover .headerlink,.rst-content h3 .headerlink:focus,.rst-content h3:hover .headerlink,.rst-content h4 .headerlink:focus,.rst-content h4:hover .headerlink,.rst-content h5 .headerlink:focus,.rst-content h5:hover .headerlink,.rst-content h6 .headerlink:focus,.rst-content h6:hover .headerlink,.rst-content p.caption .headerlink:focus,.rst-content p.caption:hover .headerlink,.rst-content p .headerlink:focus,.rst-content p:hover .headerlink,.rst-content table>caption .headerlink:focus,.rst-content table>caption:hover .headerlink{opacity:1}.rst-content p a{overflow-wrap:anywhere}.rst-content .wy-table td p,.rst-content .wy-table td ul,.rst-content .wy-table th p,.rst-content .wy-table th ul,.rst-content table.docutils td p,.rst-content table.docutils td ul,.rst-content table.docutils th p,.rst-content table.docutils th ul,.rst-content table.field-list td p,.rst-content table.field-list td ul,.rst-content table.field-list th p,.rst-content table.field-list th ul{font-size:inherit}.rst-content .btn:focus{outline:2px solid}.rst-content table>caption .headerlink:after{font-size:12px}.rst-content .centered{text-align:center}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:1px solid #e1e4e5}.rst-content .sidebar dl,.rst-content .sidebar p,.rst-content .sidebar ul{font-size:90%}.rst-content .sidebar .last,.rst-content .sidebar>:last-child{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif;font-weight:700;background:#e1e4e5;padding:6px 12px;margin:-24px -24px 24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;box-shadow:0 0 0 2px #f1c40f;display:inline;font-weight:700}.rst-content .citation-reference,.rst-content .footnote-reference{vertical-align:baseline;position:relative;top:-.4em;line-height:0;font-size:90%}.rst-content .citation-reference>span.fn-bracket,.rst-content .footnote-reference>span.fn-bracket{display:none}.rst-content .hlist{width:100%}.rst-content dl dt span.classifier:before{content:" : "}.rst-content dl dt span.classifier-delimiter{display:none!important}html.writer-html4 .rst-content table.docutils.citation,html.writer-html4 .rst-content table.docutils.footnote{background:none;border:none}html.writer-html4 .rst-content table.docutils.citation td,html.writer-html4 .rst-content table.docutils.citation tr,html.writer-html4 .rst-content table.docutils.footnote td,html.writer-html4 .rst-content table.docutils.footnote tr{border:none;background-color:transparent!important;white-space:normal}html.writer-html4 .rst-content table.docutils.citation td.label,html.writer-html4 .rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{display:grid;grid-template-columns:auto minmax(80%,95%)}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{display:inline-grid;grid-template-columns:max-content auto}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{display:grid;grid-template-columns:auto auto minmax(.65rem,auto) minmax(40%,95%)}html.writer-html5 .rst-content aside.citation>span.label,html.writer-html5 .rst-content aside.footnote>span.label,html.writer-html5 .rst-content div.citation>span.label{grid-column-start:1;grid-column-end:2}html.writer-html5 .rst-content aside.citation>span.backrefs,html.writer-html5 .rst-content aside.footnote>span.backrefs,html.writer-html5 .rst-content div.citation>span.backrefs{grid-column-start:2;grid-column-end:3;grid-row-start:1;grid-row-end:3}html.writer-html5 .rst-content aside.citation>p,html.writer-html5 .rst-content aside.footnote>p,html.writer-html5 .rst-content div.citation>p{grid-column-start:4;grid-column-end:5}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{margin-bottom:24px}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{padding-left:1rem}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dd,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dd,html.writer-html5 .rst-content dl.footnote>dt{margin-bottom:0}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{font-size:.9rem}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.footnote>dt{margin:0 .5rem .5rem 0;line-height:1.2rem;word-break:break-all;font-weight:400}html.writer-html5 .rst-content dl.citation>dt>span.brackets:before,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:before{content:"["}html.writer-html5 .rst-content dl.citation>dt>span.brackets:after,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:after{content:"]"}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a{word-break:keep-all}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a:not(:first-child):before,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.footnote>dd{margin:0 0 .5rem;line-height:1.2rem}html.writer-html5 .rst-content dl.citation>dd p,html.writer-html5 .rst-content dl.footnote>dd p{font-size:.9rem}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{padding-left:1rem;padding-right:1rem;font-size:.9rem;line-height:1.2rem}html.writer-html5 .rst-content aside.citation p,html.writer-html5 .rst-content aside.footnote p,html.writer-html5 .rst-content div.citation p{font-size:.9rem;line-height:1.2rem;margin-bottom:12px}html.writer-html5 .rst-content aside.citation span.backrefs,html.writer-html5 .rst-content aside.footnote span.backrefs,html.writer-html5 .rst-content div.citation span.backrefs{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content aside.citation span.backrefs>a,html.writer-html5 .rst-content aside.footnote span.backrefs>a,html.writer-html5 .rst-content div.citation span.backrefs>a{word-break:keep-all}html.writer-html5 .rst-content aside.citation span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content aside.footnote span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content div.citation span.backrefs>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content aside.citation span.label,html.writer-html5 .rst-content aside.footnote span.label,html.writer-html5 .rst-content div.citation span.label{line-height:1.2rem}html.writer-html5 .rst-content aside.citation-list,html.writer-html5 .rst-content aside.footnote-list,html.writer-html5 .rst-content div.citation-list{margin-bottom:24px}html.writer-html5 .rst-content dl.option-list kbd{font-size:.9rem}.rst-content table.docutils.footnote,html.writer-html4 .rst-content table.docutils.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content aside.footnote-list aside.footnote,html.writer-html5 .rst-content div.citation-list>div.citation,html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{color:grey}.rst-content table.docutils.footnote code,.rst-content table.docutils.footnote tt,html.writer-html4 .rst-content table.docutils.citation code,html.writer-html4 .rst-content table.docutils.citation tt,html.writer-html5 .rst-content aside.footnote-list aside.footnote code,html.writer-html5 .rst-content aside.footnote-list aside.footnote tt,html.writer-html5 .rst-content aside.footnote code,html.writer-html5 .rst-content aside.footnote tt,html.writer-html5 .rst-content div.citation-list>div.citation code,html.writer-html5 .rst-content div.citation-list>div.citation tt,html.writer-html5 .rst-content dl.citation code,html.writer-html5 .rst-content dl.citation tt,html.writer-html5 .rst-content dl.footnote code,html.writer-html5 .rst-content dl.footnote tt{color:#555}.rst-content .wy-table-responsive.citation,.rst-content .wy-table-responsive.footnote{margin-bottom:0}.rst-content .wy-table-responsive.citation+:not(.citation),.rst-content .wy-table-responsive.footnote+:not(.footnote){margin-top:24px}.rst-content .wy-table-responsive.citation:last-child,.rst-content .wy-table-responsive.footnote:last-child{margin-bottom:24px}.rst-content table.docutils th{border-color:#e1e4e5}html.writer-html5 .rst-content table.docutils th{border:1px solid #e1e4e5}html.writer-html5 .rst-content table.docutils td>p,html.writer-html5 .rst-content table.docutils th>p{line-height:1rem;margin-bottom:0;font-size:.9rem}.rst-content table.docutils td .last,.rst-content table.docutils td .last>:last-child{margin-bottom:0}.rst-content table.field-list,.rst-content table.field-list td{border:none}.rst-content table.field-list td p{line-height:inherit}.rst-content table.field-list td>strong{display:inline-block}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left}.rst-content code,.rst-content tt{color:#000;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;padding:2px 5px}.rst-content code big,.rst-content code em,.rst-content tt big,.rst-content tt em{font-size:100%!important;line-height:normal}.rst-content code.literal,.rst-content tt.literal{color:#e74c3c;white-space:normal}.rst-content code.xref,.rst-content tt.xref,a .rst-content code,a .rst-content tt{font-weight:700;color:#404040;overflow-wrap:normal}.rst-content kbd,.rst-content pre,.rst-content samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace}.rst-content a code,.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:700;margin-bottom:12px}.rst-content dl ol,.rst-content dl p,.rst-content dl table,.rst-content dl ul{margin-bottom:12px}.rst-content dl dd{margin:0 0 12px 24px;line-height:24px}.rst-content dl dd>ol:last-child,.rst-content dl dd>p:last-child,.rst-content dl dd>table:last-child,.rst-content dl dd>ul:last-child{margin-bottom:0}html.writer-html4 .rst-content dl:not(.docutils),html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple){margin-bottom:24px}html.writer-html4 .rst-content dl:not(.docutils)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{display:table;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:3px solid #6ab0de;padding:6px;position:relative}html.writer-html4 .rst-content dl:not(.docutils)>dt:before,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:before{color:#6ab0de}html.writer-html4 .rst-content dl:not(.docutils)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{margin-bottom:6px;border:none;border-left:3px solid #ccc;background:#f0f0f0;color:#555}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils)>dt:first-child,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:first-child{margin-top:0}html.writer-html4 .rst-content dl:not(.docutils) code.descclassname,html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descclassname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{background-color:transparent;border:none;padding:0;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .optional,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .property,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .property{display:inline-block;padding-right:8px;max-width:100%}html.writer-html4 .rst-content dl:not(.docutils) .k,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .k{font-style:italic}html.writer-html4 .rst-content dl:not(.docutils) .descclassname,html.writer-html4 .rst-content dl:not(.docutils) .descname,html.writer-html4 .rst-content dl:not(.docutils) .sig-name,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .sig-name{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#000}.rst-content .viewcode-back,.rst-content .viewcode-link{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right}.rst-content p.rubric{margin-bottom:12px;font-weight:700}.rst-content code.download,.rst-content tt.download{background:inherit;padding:inherit;font-weight:400;font-family:inherit;font-size:inherit;color:inherit;border:inherit;white-space:inherit}.rst-content code.download span:first-child,.rst-content tt.download span:first-child{-webkit-font-smoothing:subpixel-antialiased}.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{margin-right:4px}.rst-content .guilabel,.rst-content .menuselection{font-size:80%;font-weight:700;border-radius:4px;padding:2.4px 6px;margin:auto 2px}.rst-content .guilabel,.rst-content .menuselection{border:1px solid #7fbbe3;background:#e7f2fa}.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>.kbd,.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>kbd{color:inherit;font-size:80%;background-color:#fff;border:1px solid #a6a6a6;border-radius:4px;box-shadow:0 2px grey;padding:2.4px 6px;margin:auto 0}.rst-content .versionmodified{font-style:italic}@media screen and (max-width:480px){.rst-content .sidebar{width:100%}}span[id*=MathJax-Span]{color:#404040}.math{text-align:center}@font-face{font-family:Lato;src:url(fonts/lato-normal.woff2?bd03a2cc277bbbc338d464e679fe9942) format("woff2"),url(fonts/lato-normal.woff?27bd77b9162d388cb8d4c4217c7c5e2a) format("woff");font-weight:400;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold.woff2?cccb897485813c7c256901dbca54ecf2) format("woff2"),url(fonts/lato-bold.woff?d878b6c29b10beca227e9eef4246111b) format("woff");font-weight:700;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold-italic.woff2?0b6bb6725576b072c5d0b02ecdd1900d) format("woff2"),url(fonts/lato-bold-italic.woff?9c7e4e9eb485b4a121c760e61bc3707c) format("woff");font-weight:700;font-style:italic;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-normal-italic.woff2?4eb103b4d12be57cb1d040ed5e162e9d) format("woff2"),url(fonts/lato-normal-italic.woff?f28f2d6482446544ef1ea1ccc6dd5892) format("woff");font-weight:400;font-style:italic;font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:400;src:url(fonts/Roboto-Slab-Regular.woff2?7abf5b8d04d26a2cafea937019bca958) format("woff2"),url(fonts/Roboto-Slab-Regular.woff?c1be9284088d487c5e3ff0a10a92e58c) format("woff");font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:700;src:url(fonts/Roboto-Slab-Bold.woff2?9984f4a9bda09be08e83f2506954adbe) format("woff2"),url(fonts/Roboto-Slab-Bold.woff?bed5564a116b05148e3b3bea6fb1162a) format("woff");font-display:block}
\ No newline at end of file
+ */@font-face{font-family:FontAwesome;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713);src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix&v=4.7.0) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#fontawesomeregular) format("svg");font-weight:400;font-style:normal}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14286em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14286em;width:2.14286em;top:.14286em;text-align:center}.fa-li.fa-lg{left:-1.85714em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa-pull-left.icon,.fa.fa-pull-left,.rst-content .code-block-caption .fa-pull-left.headerlink,.rst-content .eqno .fa-pull-left.headerlink,.rst-content .fa-pull-left.admonition-title,.rst-content code.download span.fa-pull-left:first-child,.rst-content dl dt .fa-pull-left.headerlink,.rst-content h1 .fa-pull-left.headerlink,.rst-content h2 .fa-pull-left.headerlink,.rst-content h3 .fa-pull-left.headerlink,.rst-content h4 .fa-pull-left.headerlink,.rst-content h5 .fa-pull-left.headerlink,.rst-content h6 .fa-pull-left.headerlink,.rst-content p .fa-pull-left.headerlink,.rst-content table>caption .fa-pull-left.headerlink,.rst-content tt.download span.fa-pull-left:first-child,.wy-menu-vertical li.current>a button.fa-pull-left.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-left.toctree-expand,.wy-menu-vertical li button.fa-pull-left.toctree-expand{margin-right:.3em}.fa-pull-right.icon,.fa.fa-pull-right,.rst-content .code-block-caption .fa-pull-right.headerlink,.rst-content .eqno .fa-pull-right.headerlink,.rst-content .fa-pull-right.admonition-title,.rst-content code.download span.fa-pull-right:first-child,.rst-content dl dt .fa-pull-right.headerlink,.rst-content h1 .fa-pull-right.headerlink,.rst-content h2 .fa-pull-right.headerlink,.rst-content h3 .fa-pull-right.headerlink,.rst-content h4 .fa-pull-right.headerlink,.rst-content h5 .fa-pull-right.headerlink,.rst-content h6 .fa-pull-right.headerlink,.rst-content p .fa-pull-right.headerlink,.rst-content table>caption .fa-pull-right.headerlink,.rst-content tt.download span.fa-pull-right:first-child,.wy-menu-vertical li.current>a button.fa-pull-right.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-right.toctree-expand,.wy-menu-vertical li button.fa-pull-right.toctree-expand{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left,.pull-left.icon,.rst-content .code-block-caption .pull-left.headerlink,.rst-content .eqno .pull-left.headerlink,.rst-content .pull-left.admonition-title,.rst-content code.download span.pull-left:first-child,.rst-content dl dt .pull-left.headerlink,.rst-content h1 .pull-left.headerlink,.rst-content h2 .pull-left.headerlink,.rst-content h3 .pull-left.headerlink,.rst-content h4 .pull-left.headerlink,.rst-content h5 .pull-left.headerlink,.rst-content h6 .pull-left.headerlink,.rst-content p .pull-left.headerlink,.rst-content table>caption .pull-left.headerlink,.rst-content tt.download span.pull-left:first-child,.wy-menu-vertical li.current>a button.pull-left.toctree-expand,.wy-menu-vertical li.on a button.pull-left.toctree-expand,.wy-menu-vertical li button.pull-left.toctree-expand{margin-right:.3em}.fa.pull-right,.pull-right.icon,.rst-content .code-block-caption .pull-right.headerlink,.rst-content .eqno .pull-right.headerlink,.rst-content .pull-right.admonition-title,.rst-content code.download span.pull-right:first-child,.rst-content dl dt .pull-right.headerlink,.rst-content h1 .pull-right.headerlink,.rst-content h2 .pull-right.headerlink,.rst-content h3 .pull-right.headerlink,.rst-content h4 .pull-right.headerlink,.rst-content h5 .pull-right.headerlink,.rst-content h6 .pull-right.headerlink,.rst-content p .pull-right.headerlink,.rst-content table>caption .pull-right.headerlink,.rst-content tt.download span.pull-right:first-child,.wy-menu-vertical li.current>a button.pull-right.toctree-expand,.wy-menu-vertical li.on a button.pull-right.toctree-expand,.wy-menu-vertical li button.pull-right.toctree-expand{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s linear infinite;animation:fa-spin 2s linear infinite}.fa-pulse{-webkit-animation:fa-spin 1s steps(8) infinite;animation:fa-spin 1s steps(8) infinite}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scaleX(-1);-ms-transform:scaleX(-1);transform:scaleX(-1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scaleY(-1);-ms-transform:scaleY(-1);transform:scaleY(-1)}:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:""}.fa-music:before{content:""}.fa-search:before,.icon-search:before{content:""}.fa-envelope-o:before{content:""}.fa-heart:before{content:""}.fa-star:before{content:""}.fa-star-o:before{content:""}.fa-user:before{content:""}.fa-film:before{content:""}.fa-th-large:before{content:""}.fa-th:before{content:""}.fa-th-list:before{content:""}.fa-check:before{content:""}.fa-close:before,.fa-remove:before,.fa-times:before{content:""}.fa-search-plus:before{content:""}.fa-search-minus:before{content:""}.fa-power-off:before{content:""}.fa-signal:before{content:""}.fa-cog:before,.fa-gear:before{content:""}.fa-trash-o:before{content:""}.fa-home:before,.icon-home:before{content:""}.fa-file-o:before{content:""}.fa-clock-o:before{content:""}.fa-road:before{content:""}.fa-download:before,.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{content:""}.fa-arrow-circle-o-down:before{content:""}.fa-arrow-circle-o-up:before{content:""}.fa-inbox:before{content:""}.fa-play-circle-o:before{content:""}.fa-repeat:before,.fa-rotate-right:before{content:""}.fa-refresh:before{content:""}.fa-list-alt:before{content:""}.fa-lock:before{content:""}.fa-flag:before{content:""}.fa-headphones:before{content:""}.fa-volume-off:before{content:""}.fa-volume-down:before{content:""}.fa-volume-up:before{content:""}.fa-qrcode:before{content:""}.fa-barcode:before{content:""}.fa-tag:before{content:""}.fa-tags:before{content:""}.fa-book:before,.icon-book:before{content:""}.fa-bookmark:before{content:""}.fa-print:before{content:""}.fa-camera:before{content:""}.fa-font:before{content:""}.fa-bold:before{content:""}.fa-italic:before{content:""}.fa-text-height:before{content:""}.fa-text-width:before{content:""}.fa-align-left:before{content:""}.fa-align-center:before{content:""}.fa-align-right:before{content:""}.fa-align-justify:before{content:""}.fa-list:before{content:""}.fa-dedent:before,.fa-outdent:before{content:""}.fa-indent:before{content:""}.fa-video-camera:before{content:""}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:""}.fa-pencil:before{content:""}.fa-map-marker:before{content:""}.fa-adjust:before{content:""}.fa-tint:before{content:""}.fa-edit:before,.fa-pencil-square-o:before{content:""}.fa-share-square-o:before{content:""}.fa-check-square-o:before{content:""}.fa-arrows:before{content:""}.fa-step-backward:before{content:""}.fa-fast-backward:before{content:""}.fa-backward:before{content:""}.fa-play:before{content:""}.fa-pause:before{content:""}.fa-stop:before{content:""}.fa-forward:before{content:""}.fa-fast-forward:before{content:""}.fa-step-forward:before{content:""}.fa-eject:before{content:""}.fa-chevron-left:before{content:""}.fa-chevron-right:before{content:""}.fa-plus-circle:before{content:""}.fa-minus-circle:before{content:""}.fa-times-circle:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before{content:""}.fa-check-circle:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before{content:""}.fa-question-circle:before{content:""}.fa-info-circle:before{content:""}.fa-crosshairs:before{content:""}.fa-times-circle-o:before{content:""}.fa-check-circle-o:before{content:""}.fa-ban:before{content:""}.fa-arrow-left:before{content:""}.fa-arrow-right:before{content:""}.fa-arrow-up:before{content:""}.fa-arrow-down:before{content:""}.fa-mail-forward:before,.fa-share:before{content:""}.fa-expand:before{content:""}.fa-compress:before{content:""}.fa-plus:before{content:""}.fa-minus:before{content:""}.fa-asterisk:before{content:""}.fa-exclamation-circle:before,.rst-content .admonition-title:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before{content:""}.fa-gift:before{content:""}.fa-leaf:before{content:""}.fa-fire:before,.icon-fire:before{content:""}.fa-eye:before{content:""}.fa-eye-slash:before{content:""}.fa-exclamation-triangle:before,.fa-warning:before{content:""}.fa-plane:before{content:""}.fa-calendar:before{content:""}.fa-random:before{content:""}.fa-comment:before{content:""}.fa-magnet:before{content:""}.fa-chevron-up:before{content:""}.fa-chevron-down:before{content:""}.fa-retweet:before{content:""}.fa-shopping-cart:before{content:""}.fa-folder:before{content:""}.fa-folder-open:before{content:""}.fa-arrows-v:before{content:""}.fa-arrows-h:before{content:""}.fa-bar-chart-o:before,.fa-bar-chart:before{content:""}.fa-twitter-square:before{content:""}.fa-facebook-square:before{content:""}.fa-camera-retro:before{content:""}.fa-key:before{content:""}.fa-cogs:before,.fa-gears:before{content:""}.fa-comments:before{content:""}.fa-thumbs-o-up:before{content:""}.fa-thumbs-o-down:before{content:""}.fa-star-half:before{content:""}.fa-heart-o:before{content:""}.fa-sign-out:before{content:""}.fa-linkedin-square:before{content:""}.fa-thumb-tack:before{content:""}.fa-external-link:before{content:""}.fa-sign-in:before{content:""}.fa-trophy:before{content:""}.fa-github-square:before{content:""}.fa-upload:before{content:""}.fa-lemon-o:before{content:""}.fa-phone:before{content:""}.fa-square-o:before{content:""}.fa-bookmark-o:before{content:""}.fa-phone-square:before{content:""}.fa-twitter:before{content:""}.fa-facebook-f:before,.fa-facebook:before{content:""}.fa-github:before,.icon-github:before{content:""}.fa-unlock:before{content:""}.fa-credit-card:before{content:""}.fa-feed:before,.fa-rss:before{content:""}.fa-hdd-o:before{content:""}.fa-bullhorn:before{content:""}.fa-bell:before{content:""}.fa-certificate:before{content:""}.fa-hand-o-right:before{content:""}.fa-hand-o-left:before{content:""}.fa-hand-o-up:before{content:""}.fa-hand-o-down:before{content:""}.fa-arrow-circle-left:before,.icon-circle-arrow-left:before{content:""}.fa-arrow-circle-right:before,.icon-circle-arrow-right:before{content:""}.fa-arrow-circle-up:before{content:""}.fa-arrow-circle-down:before{content:""}.fa-globe:before{content:""}.fa-wrench:before{content:""}.fa-tasks:before{content:""}.fa-filter:before{content:""}.fa-briefcase:before{content:""}.fa-arrows-alt:before{content:""}.fa-group:before,.fa-users:before{content:""}.fa-chain:before,.fa-link:before,.icon-link:before{content:""}.fa-cloud:before{content:""}.fa-flask:before{content:""}.fa-cut:before,.fa-scissors:before{content:""}.fa-copy:before,.fa-files-o:before{content:""}.fa-paperclip:before{content:""}.fa-floppy-o:before,.fa-save:before{content:""}.fa-square:before{content:""}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:""}.fa-list-ul:before{content:""}.fa-list-ol:before{content:""}.fa-strikethrough:before{content:""}.fa-underline:before{content:""}.fa-table:before{content:""}.fa-magic:before{content:""}.fa-truck:before{content:""}.fa-pinterest:before{content:""}.fa-pinterest-square:before{content:""}.fa-google-plus-square:before{content:""}.fa-google-plus:before{content:""}.fa-money:before{content:""}.fa-caret-down:before,.icon-caret-down:before,.wy-dropdown .caret:before{content:""}.fa-caret-up:before{content:""}.fa-caret-left:before{content:""}.fa-caret-right:before{content:""}.fa-columns:before{content:""}.fa-sort:before,.fa-unsorted:before{content:""}.fa-sort-desc:before,.fa-sort-down:before{content:""}.fa-sort-asc:before,.fa-sort-up:before{content:""}.fa-envelope:before{content:""}.fa-linkedin:before{content:""}.fa-rotate-left:before,.fa-undo:before{content:""}.fa-gavel:before,.fa-legal:before{content:""}.fa-dashboard:before,.fa-tachometer:before{content:""}.fa-comment-o:before{content:""}.fa-comments-o:before{content:""}.fa-bolt:before,.fa-flash:before{content:""}.fa-sitemap:before{content:""}.fa-umbrella:before{content:""}.fa-clipboard:before,.fa-paste:before{content:""}.fa-lightbulb-o:before{content:""}.fa-exchange:before{content:""}.fa-cloud-download:before{content:""}.fa-cloud-upload:before{content:""}.fa-user-md:before{content:""}.fa-stethoscope:before{content:""}.fa-suitcase:before{content:""}.fa-bell-o:before{content:""}.fa-coffee:before{content:""}.fa-cutlery:before{content:""}.fa-file-text-o:before{content:""}.fa-building-o:before{content:""}.fa-hospital-o:before{content:""}.fa-ambulance:before{content:""}.fa-medkit:before{content:""}.fa-fighter-jet:before{content:""}.fa-beer:before{content:""}.fa-h-square:before{content:""}.fa-plus-square:before{content:""}.fa-angle-double-left:before{content:""}.fa-angle-double-right:before{content:""}.fa-angle-double-up:before{content:""}.fa-angle-double-down:before{content:""}.fa-angle-left:before{content:""}.fa-angle-right:before{content:""}.fa-angle-up:before{content:""}.fa-angle-down:before{content:""}.fa-desktop:before{content:""}.fa-laptop:before{content:""}.fa-tablet:before{content:""}.fa-mobile-phone:before,.fa-mobile:before{content:""}.fa-circle-o:before{content:""}.fa-quote-left:before{content:""}.fa-quote-right:before{content:""}.fa-spinner:before{content:""}.fa-circle:before{content:""}.fa-mail-reply:before,.fa-reply:before{content:""}.fa-github-alt:before{content:""}.fa-folder-o:before{content:""}.fa-folder-open-o:before{content:""}.fa-smile-o:before{content:""}.fa-frown-o:before{content:""}.fa-meh-o:before{content:""}.fa-gamepad:before{content:""}.fa-keyboard-o:before{content:""}.fa-flag-o:before{content:""}.fa-flag-checkered:before{content:""}.fa-terminal:before{content:""}.fa-code:before{content:""}.fa-mail-reply-all:before,.fa-reply-all:before{content:""}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:""}.fa-location-arrow:before{content:""}.fa-crop:before{content:""}.fa-code-fork:before{content:""}.fa-chain-broken:before,.fa-unlink:before{content:""}.fa-question:before{content:""}.fa-info:before{content:""}.fa-exclamation:before{content:""}.fa-superscript:before{content:""}.fa-subscript:before{content:""}.fa-eraser:before{content:""}.fa-puzzle-piece:before{content:""}.fa-microphone:before{content:""}.fa-microphone-slash:before{content:""}.fa-shield:before{content:""}.fa-calendar-o:before{content:""}.fa-fire-extinguisher:before{content:""}.fa-rocket:before{content:""}.fa-maxcdn:before{content:""}.fa-chevron-circle-left:before{content:""}.fa-chevron-circle-right:before{content:""}.fa-chevron-circle-up:before{content:""}.fa-chevron-circle-down:before{content:""}.fa-html5:before{content:""}.fa-css3:before{content:""}.fa-anchor:before{content:""}.fa-unlock-alt:before{content:""}.fa-bullseye:before{content:""}.fa-ellipsis-h:before{content:""}.fa-ellipsis-v:before{content:""}.fa-rss-square:before{content:""}.fa-play-circle:before{content:""}.fa-ticket:before{content:""}.fa-minus-square:before{content:""}.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before{content:""}.fa-level-up:before{content:""}.fa-level-down:before{content:""}.fa-check-square:before{content:""}.fa-pencil-square:before{content:""}.fa-external-link-square:before{content:""}.fa-share-square:before{content:""}.fa-compass:before{content:""}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:""}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:""}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:""}.fa-eur:before,.fa-euro:before{content:""}.fa-gbp:before{content:""}.fa-dollar:before,.fa-usd:before{content:""}.fa-inr:before,.fa-rupee:before{content:""}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:""}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:""}.fa-krw:before,.fa-won:before{content:""}.fa-bitcoin:before,.fa-btc:before{content:""}.fa-file:before{content:""}.fa-file-text:before{content:""}.fa-sort-alpha-asc:before{content:""}.fa-sort-alpha-desc:before{content:""}.fa-sort-amount-asc:before{content:""}.fa-sort-amount-desc:before{content:""}.fa-sort-numeric-asc:before{content:""}.fa-sort-numeric-desc:before{content:""}.fa-thumbs-up:before{content:""}.fa-thumbs-down:before{content:""}.fa-youtube-square:before{content:""}.fa-youtube:before{content:""}.fa-xing:before{content:""}.fa-xing-square:before{content:""}.fa-youtube-play:before{content:""}.fa-dropbox:before{content:""}.fa-stack-overflow:before{content:""}.fa-instagram:before{content:""}.fa-flickr:before{content:""}.fa-adn:before{content:""}.fa-bitbucket:before,.icon-bitbucket:before{content:""}.fa-bitbucket-square:before{content:""}.fa-tumblr:before{content:""}.fa-tumblr-square:before{content:""}.fa-long-arrow-down:before{content:""}.fa-long-arrow-up:before{content:""}.fa-long-arrow-left:before{content:""}.fa-long-arrow-right:before{content:""}.fa-apple:before{content:""}.fa-windows:before{content:""}.fa-android:before{content:""}.fa-linux:before{content:""}.fa-dribbble:before{content:""}.fa-skype:before{content:""}.fa-foursquare:before{content:""}.fa-trello:before{content:""}.fa-female:before{content:""}.fa-male:before{content:""}.fa-gittip:before,.fa-gratipay:before{content:""}.fa-sun-o:before{content:""}.fa-moon-o:before{content:""}.fa-archive:before{content:""}.fa-bug:before{content:""}.fa-vk:before{content:""}.fa-weibo:before{content:""}.fa-renren:before{content:""}.fa-pagelines:before{content:""}.fa-stack-exchange:before{content:""}.fa-arrow-circle-o-right:before{content:""}.fa-arrow-circle-o-left:before{content:""}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:""}.fa-dot-circle-o:before{content:""}.fa-wheelchair:before{content:""}.fa-vimeo-square:before{content:""}.fa-try:before,.fa-turkish-lira:before{content:""}.fa-plus-square-o:before,.wy-menu-vertical li button.toctree-expand:before{content:""}.fa-space-shuttle:before{content:""}.fa-slack:before{content:""}.fa-envelope-square:before{content:""}.fa-wordpress:before{content:""}.fa-openid:before{content:""}.fa-bank:before,.fa-institution:before,.fa-university:before{content:""}.fa-graduation-cap:before,.fa-mortar-board:before{content:""}.fa-yahoo:before{content:""}.fa-google:before{content:""}.fa-reddit:before{content:""}.fa-reddit-square:before{content:""}.fa-stumbleupon-circle:before{content:""}.fa-stumbleupon:before{content:""}.fa-delicious:before{content:""}.fa-digg:before{content:""}.fa-pied-piper-pp:before{content:""}.fa-pied-piper-alt:before{content:""}.fa-drupal:before{content:""}.fa-joomla:before{content:""}.fa-language:before{content:""}.fa-fax:before{content:""}.fa-building:before{content:""}.fa-child:before{content:""}.fa-paw:before{content:""}.fa-spoon:before{content:""}.fa-cube:before{content:""}.fa-cubes:before{content:""}.fa-behance:before{content:""}.fa-behance-square:before{content:""}.fa-steam:before{content:""}.fa-steam-square:before{content:""}.fa-recycle:before{content:""}.fa-automobile:before,.fa-car:before{content:""}.fa-cab:before,.fa-taxi:before{content:""}.fa-tree:before{content:""}.fa-spotify:before{content:""}.fa-deviantart:before{content:""}.fa-soundcloud:before{content:""}.fa-database:before{content:""}.fa-file-pdf-o:before{content:""}.fa-file-word-o:before{content:""}.fa-file-excel-o:before{content:""}.fa-file-powerpoint-o:before{content:""}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:""}.fa-file-archive-o:before,.fa-file-zip-o:before{content:""}.fa-file-audio-o:before,.fa-file-sound-o:before{content:""}.fa-file-movie-o:before,.fa-file-video-o:before{content:""}.fa-file-code-o:before{content:""}.fa-vine:before{content:""}.fa-codepen:before{content:""}.fa-jsfiddle:before{content:""}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:""}.fa-circle-o-notch:before{content:""}.fa-ra:before,.fa-rebel:before,.fa-resistance:before{content:""}.fa-empire:before,.fa-ge:before{content:""}.fa-git-square:before{content:""}.fa-git:before{content:""}.fa-hacker-news:before,.fa-y-combinator-square:before,.fa-yc-square:before{content:""}.fa-tencent-weibo:before{content:""}.fa-qq:before{content:""}.fa-wechat:before,.fa-weixin:before{content:""}.fa-paper-plane:before,.fa-send:before{content:""}.fa-paper-plane-o:before,.fa-send-o:before{content:""}.fa-history:before{content:""}.fa-circle-thin:before{content:""}.fa-header:before{content:""}.fa-paragraph:before{content:""}.fa-sliders:before{content:""}.fa-share-alt:before{content:""}.fa-share-alt-square:before{content:""}.fa-bomb:before{content:""}.fa-futbol-o:before,.fa-soccer-ball-o:before{content:""}.fa-tty:before{content:""}.fa-binoculars:before{content:""}.fa-plug:before{content:""}.fa-slideshare:before{content:""}.fa-twitch:before{content:""}.fa-yelp:before{content:""}.fa-newspaper-o:before{content:""}.fa-wifi:before{content:""}.fa-calculator:before{content:""}.fa-paypal:before{content:""}.fa-google-wallet:before{content:""}.fa-cc-visa:before{content:""}.fa-cc-mastercard:before{content:""}.fa-cc-discover:before{content:""}.fa-cc-amex:before{content:""}.fa-cc-paypal:before{content:""}.fa-cc-stripe:before{content:""}.fa-bell-slash:before{content:""}.fa-bell-slash-o:before{content:""}.fa-trash:before{content:""}.fa-copyright:before{content:""}.fa-at:before{content:""}.fa-eyedropper:before{content:""}.fa-paint-brush:before{content:""}.fa-birthday-cake:before{content:""}.fa-area-chart:before{content:""}.fa-pie-chart:before{content:""}.fa-line-chart:before{content:""}.fa-lastfm:before{content:""}.fa-lastfm-square:before{content:""}.fa-toggle-off:before{content:""}.fa-toggle-on:before{content:""}.fa-bicycle:before{content:""}.fa-bus:before{content:""}.fa-ioxhost:before{content:""}.fa-angellist:before{content:""}.fa-cc:before{content:""}.fa-ils:before,.fa-shekel:before,.fa-sheqel:before{content:""}.fa-meanpath:before{content:""}.fa-buysellads:before{content:""}.fa-connectdevelop:before{content:""}.fa-dashcube:before{content:""}.fa-forumbee:before{content:""}.fa-leanpub:before{content:""}.fa-sellsy:before{content:""}.fa-shirtsinbulk:before{content:""}.fa-simplybuilt:before{content:""}.fa-skyatlas:before{content:""}.fa-cart-plus:before{content:""}.fa-cart-arrow-down:before{content:""}.fa-diamond:before{content:""}.fa-ship:before{content:""}.fa-user-secret:before{content:""}.fa-motorcycle:before{content:""}.fa-street-view:before{content:""}.fa-heartbeat:before{content:""}.fa-venus:before{content:""}.fa-mars:before{content:""}.fa-mercury:before{content:""}.fa-intersex:before,.fa-transgender:before{content:""}.fa-transgender-alt:before{content:""}.fa-venus-double:before{content:""}.fa-mars-double:before{content:""}.fa-venus-mars:before{content:""}.fa-mars-stroke:before{content:""}.fa-mars-stroke-v:before{content:""}.fa-mars-stroke-h:before{content:""}.fa-neuter:before{content:""}.fa-genderless:before{content:""}.fa-facebook-official:before{content:""}.fa-pinterest-p:before{content:""}.fa-whatsapp:before{content:""}.fa-server:before{content:""}.fa-user-plus:before{content:""}.fa-user-times:before{content:""}.fa-bed:before,.fa-hotel:before{content:""}.fa-viacoin:before{content:""}.fa-train:before{content:""}.fa-subway:before{content:""}.fa-medium:before{content:""}.fa-y-combinator:before,.fa-yc:before{content:""}.fa-optin-monster:before{content:""}.fa-opencart:before{content:""}.fa-expeditedssl:before{content:""}.fa-battery-4:before,.fa-battery-full:before,.fa-battery:before{content:""}.fa-battery-3:before,.fa-battery-three-quarters:before{content:""}.fa-battery-2:before,.fa-battery-half:before{content:""}.fa-battery-1:before,.fa-battery-quarter:before{content:""}.fa-battery-0:before,.fa-battery-empty:before{content:""}.fa-mouse-pointer:before{content:""}.fa-i-cursor:before{content:""}.fa-object-group:before{content:""}.fa-object-ungroup:before{content:""}.fa-sticky-note:before{content:""}.fa-sticky-note-o:before{content:""}.fa-cc-jcb:before{content:""}.fa-cc-diners-club:before{content:""}.fa-clone:before{content:""}.fa-balance-scale:before{content:""}.fa-hourglass-o:before{content:""}.fa-hourglass-1:before,.fa-hourglass-start:before{content:""}.fa-hourglass-2:before,.fa-hourglass-half:before{content:""}.fa-hourglass-3:before,.fa-hourglass-end:before{content:""}.fa-hourglass:before{content:""}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:""}.fa-hand-paper-o:before,.fa-hand-stop-o:before{content:""}.fa-hand-scissors-o:before{content:""}.fa-hand-lizard-o:before{content:""}.fa-hand-spock-o:before{content:""}.fa-hand-pointer-o:before{content:""}.fa-hand-peace-o:before{content:""}.fa-trademark:before{content:""}.fa-registered:before{content:""}.fa-creative-commons:before{content:""}.fa-gg:before{content:""}.fa-gg-circle:before{content:""}.fa-tripadvisor:before{content:""}.fa-odnoklassniki:before{content:""}.fa-odnoklassniki-square:before{content:""}.fa-get-pocket:before{content:""}.fa-wikipedia-w:before{content:""}.fa-safari:before{content:""}.fa-chrome:before{content:""}.fa-firefox:before{content:""}.fa-opera:before{content:""}.fa-internet-explorer:before{content:""}.fa-television:before,.fa-tv:before{content:""}.fa-contao:before{content:""}.fa-500px:before{content:""}.fa-amazon:before{content:""}.fa-calendar-plus-o:before{content:""}.fa-calendar-minus-o:before{content:""}.fa-calendar-times-o:before{content:""}.fa-calendar-check-o:before{content:""}.fa-industry:before{content:""}.fa-map-pin:before{content:""}.fa-map-signs:before{content:""}.fa-map-o:before{content:""}.fa-map:before{content:""}.fa-commenting:before{content:""}.fa-commenting-o:before{content:""}.fa-houzz:before{content:""}.fa-vimeo:before{content:""}.fa-black-tie:before{content:""}.fa-fonticons:before{content:""}.fa-reddit-alien:before{content:""}.fa-edge:before{content:""}.fa-credit-card-alt:before{content:""}.fa-codiepie:before{content:""}.fa-modx:before{content:""}.fa-fort-awesome:before{content:""}.fa-usb:before{content:""}.fa-product-hunt:before{content:""}.fa-mixcloud:before{content:""}.fa-scribd:before{content:""}.fa-pause-circle:before{content:""}.fa-pause-circle-o:before{content:""}.fa-stop-circle:before{content:""}.fa-stop-circle-o:before{content:""}.fa-shopping-bag:before{content:""}.fa-shopping-basket:before{content:""}.fa-hashtag:before{content:""}.fa-bluetooth:before{content:""}.fa-bluetooth-b:before{content:""}.fa-percent:before{content:""}.fa-gitlab:before,.icon-gitlab:before{content:""}.fa-wpbeginner:before{content:""}.fa-wpforms:before{content:""}.fa-envira:before{content:""}.fa-universal-access:before{content:""}.fa-wheelchair-alt:before{content:""}.fa-question-circle-o:before{content:""}.fa-blind:before{content:""}.fa-audio-description:before{content:""}.fa-volume-control-phone:before{content:""}.fa-braille:before{content:""}.fa-assistive-listening-systems:before{content:""}.fa-american-sign-language-interpreting:before,.fa-asl-interpreting:before{content:""}.fa-deaf:before,.fa-deafness:before,.fa-hard-of-hearing:before{content:""}.fa-glide:before{content:""}.fa-glide-g:before{content:""}.fa-sign-language:before,.fa-signing:before{content:""}.fa-low-vision:before{content:""}.fa-viadeo:before{content:""}.fa-viadeo-square:before{content:""}.fa-snapchat:before{content:""}.fa-snapchat-ghost:before{content:""}.fa-snapchat-square:before{content:""}.fa-pied-piper:before{content:""}.fa-first-order:before{content:""}.fa-yoast:before{content:""}.fa-themeisle:before{content:""}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:""}.fa-fa:before,.fa-font-awesome:before{content:""}.fa-handshake-o:before{content:""}.fa-envelope-open:before{content:""}.fa-envelope-open-o:before{content:""}.fa-linode:before{content:""}.fa-address-book:before{content:""}.fa-address-book-o:before{content:""}.fa-address-card:before,.fa-vcard:before{content:""}.fa-address-card-o:before,.fa-vcard-o:before{content:""}.fa-user-circle:before{content:""}.fa-user-circle-o:before{content:""}.fa-user-o:before{content:""}.fa-id-badge:before{content:""}.fa-drivers-license:before,.fa-id-card:before{content:""}.fa-drivers-license-o:before,.fa-id-card-o:before{content:""}.fa-quora:before{content:""}.fa-free-code-camp:before{content:""}.fa-telegram:before{content:""}.fa-thermometer-4:before,.fa-thermometer-full:before,.fa-thermometer:before{content:""}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:""}.fa-thermometer-2:before,.fa-thermometer-half:before{content:""}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:""}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:""}.fa-shower:before{content:""}.fa-bath:before,.fa-bathtub:before,.fa-s15:before{content:""}.fa-podcast:before{content:""}.fa-window-maximize:before{content:""}.fa-window-minimize:before{content:""}.fa-window-restore:before{content:""}.fa-times-rectangle:before,.fa-window-close:before{content:""}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:""}.fa-bandcamp:before{content:""}.fa-grav:before{content:""}.fa-etsy:before{content:""}.fa-imdb:before{content:""}.fa-ravelry:before{content:""}.fa-eercast:before{content:""}.fa-microchip:before{content:""}.fa-snowflake-o:before{content:""}.fa-superpowers:before{content:""}.fa-wpexplorer:before{content:""}.fa-meetup:before{content:""}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-dropdown .caret,.wy-inline-validate.wy-inline-validate-danger .wy-input-context,.wy-inline-validate.wy-inline-validate-info .wy-input-context,.wy-inline-validate.wy-inline-validate-success .wy-input-context,.wy-inline-validate.wy-inline-validate-warning .wy-input-context,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{font-family:inherit}.fa:before,.icon:before,.rst-content .admonition-title:before,.rst-content .code-block-caption .headerlink:before,.rst-content .eqno .headerlink:before,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before{font-family:FontAwesome;display:inline-block;font-style:normal;font-weight:400;line-height:1;text-decoration:inherit}.rst-content .code-block-caption a .headerlink,.rst-content .eqno a .headerlink,.rst-content a .admonition-title,.rst-content code.download a span:first-child,.rst-content dl dt a .headerlink,.rst-content h1 a .headerlink,.rst-content h2 a .headerlink,.rst-content h3 a .headerlink,.rst-content h4 a .headerlink,.rst-content h5 a .headerlink,.rst-content h6 a .headerlink,.rst-content p.caption a .headerlink,.rst-content p a .headerlink,.rst-content table>caption a .headerlink,.rst-content tt.download a span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li a button.toctree-expand,a .fa,a .icon,a .rst-content .admonition-title,a .rst-content .code-block-caption .headerlink,a .rst-content .eqno .headerlink,a .rst-content code.download span:first-child,a .rst-content dl dt .headerlink,a .rst-content h1 .headerlink,a .rst-content h2 .headerlink,a .rst-content h3 .headerlink,a .rst-content h4 .headerlink,a .rst-content h5 .headerlink,a .rst-content h6 .headerlink,a .rst-content p.caption .headerlink,a .rst-content p .headerlink,a .rst-content table>caption .headerlink,a .rst-content tt.download span:first-child,a .wy-menu-vertical li button.toctree-expand{display:inline-block;text-decoration:inherit}.btn .fa,.btn .icon,.btn .rst-content .admonition-title,.btn .rst-content .code-block-caption .headerlink,.btn .rst-content .eqno .headerlink,.btn .rst-content code.download span:first-child,.btn .rst-content dl dt .headerlink,.btn .rst-content h1 .headerlink,.btn .rst-content h2 .headerlink,.btn .rst-content h3 .headerlink,.btn .rst-content h4 .headerlink,.btn .rst-content h5 .headerlink,.btn .rst-content h6 .headerlink,.btn .rst-content p .headerlink,.btn .rst-content table>caption .headerlink,.btn .rst-content tt.download span:first-child,.btn .wy-menu-vertical li.current>a button.toctree-expand,.btn .wy-menu-vertical li.on a button.toctree-expand,.btn .wy-menu-vertical li button.toctree-expand,.nav .fa,.nav .icon,.nav .rst-content .admonition-title,.nav .rst-content .code-block-caption .headerlink,.nav .rst-content .eqno .headerlink,.nav .rst-content code.download span:first-child,.nav .rst-content dl dt .headerlink,.nav .rst-content h1 .headerlink,.nav .rst-content h2 .headerlink,.nav .rst-content h3 .headerlink,.nav .rst-content h4 .headerlink,.nav .rst-content h5 .headerlink,.nav .rst-content h6 .headerlink,.nav .rst-content p .headerlink,.nav .rst-content table>caption .headerlink,.nav .rst-content tt.download span:first-child,.nav .wy-menu-vertical li.current>a button.toctree-expand,.nav .wy-menu-vertical li.on a button.toctree-expand,.nav .wy-menu-vertical li button.toctree-expand,.rst-content .btn .admonition-title,.rst-content .code-block-caption .btn .headerlink,.rst-content .code-block-caption .nav .headerlink,.rst-content .eqno .btn .headerlink,.rst-content .eqno .nav .headerlink,.rst-content .nav .admonition-title,.rst-content code.download .btn span:first-child,.rst-content code.download .nav span:first-child,.rst-content dl dt .btn .headerlink,.rst-content dl dt .nav .headerlink,.rst-content h1 .btn .headerlink,.rst-content h1 .nav .headerlink,.rst-content h2 .btn .headerlink,.rst-content h2 .nav .headerlink,.rst-content h3 .btn .headerlink,.rst-content h3 .nav .headerlink,.rst-content h4 .btn .headerlink,.rst-content h4 .nav .headerlink,.rst-content h5 .btn .headerlink,.rst-content h5 .nav .headerlink,.rst-content h6 .btn .headerlink,.rst-content h6 .nav .headerlink,.rst-content p .btn .headerlink,.rst-content p .nav .headerlink,.rst-content table>caption .btn .headerlink,.rst-content table>caption .nav .headerlink,.rst-content tt.download .btn span:first-child,.rst-content tt.download .nav span:first-child,.wy-menu-vertical li .btn button.toctree-expand,.wy-menu-vertical li.current>a .btn button.toctree-expand,.wy-menu-vertical li.current>a .nav button.toctree-expand,.wy-menu-vertical li .nav button.toctree-expand,.wy-menu-vertical li.on a .btn button.toctree-expand,.wy-menu-vertical li.on a .nav button.toctree-expand{display:inline}.btn .fa-large.icon,.btn .fa.fa-large,.btn .rst-content .code-block-caption .fa-large.headerlink,.btn .rst-content .eqno .fa-large.headerlink,.btn .rst-content .fa-large.admonition-title,.btn .rst-content code.download span.fa-large:first-child,.btn .rst-content dl dt .fa-large.headerlink,.btn .rst-content h1 .fa-large.headerlink,.btn .rst-content h2 .fa-large.headerlink,.btn .rst-content h3 .fa-large.headerlink,.btn .rst-content h4 .fa-large.headerlink,.btn .rst-content h5 .fa-large.headerlink,.btn .rst-content h6 .fa-large.headerlink,.btn .rst-content p .fa-large.headerlink,.btn .rst-content table>caption .fa-large.headerlink,.btn .rst-content tt.download span.fa-large:first-child,.btn .wy-menu-vertical li button.fa-large.toctree-expand,.nav .fa-large.icon,.nav .fa.fa-large,.nav .rst-content .code-block-caption .fa-large.headerlink,.nav .rst-content .eqno .fa-large.headerlink,.nav .rst-content .fa-large.admonition-title,.nav .rst-content code.download span.fa-large:first-child,.nav .rst-content dl dt .fa-large.headerlink,.nav .rst-content h1 .fa-large.headerlink,.nav .rst-content h2 .fa-large.headerlink,.nav .rst-content h3 .fa-large.headerlink,.nav .rst-content h4 .fa-large.headerlink,.nav .rst-content h5 .fa-large.headerlink,.nav .rst-content h6 .fa-large.headerlink,.nav .rst-content p .fa-large.headerlink,.nav .rst-content table>caption .fa-large.headerlink,.nav .rst-content tt.download span.fa-large:first-child,.nav .wy-menu-vertical li button.fa-large.toctree-expand,.rst-content .btn .fa-large.admonition-title,.rst-content .code-block-caption .btn .fa-large.headerlink,.rst-content .code-block-caption .nav .fa-large.headerlink,.rst-content .eqno .btn .fa-large.headerlink,.rst-content .eqno .nav .fa-large.headerlink,.rst-content .nav .fa-large.admonition-title,.rst-content code.download .btn span.fa-large:first-child,.rst-content code.download .nav span.fa-large:first-child,.rst-content dl dt .btn .fa-large.headerlink,.rst-content dl dt .nav .fa-large.headerlink,.rst-content h1 .btn .fa-large.headerlink,.rst-content h1 .nav .fa-large.headerlink,.rst-content h2 .btn .fa-large.headerlink,.rst-content h2 .nav .fa-large.headerlink,.rst-content h3 .btn .fa-large.headerlink,.rst-content h3 .nav .fa-large.headerlink,.rst-content h4 .btn .fa-large.headerlink,.rst-content h4 .nav .fa-large.headerlink,.rst-content h5 .btn .fa-large.headerlink,.rst-content h5 .nav .fa-large.headerlink,.rst-content h6 .btn .fa-large.headerlink,.rst-content h6 .nav .fa-large.headerlink,.rst-content p .btn .fa-large.headerlink,.rst-content p .nav .fa-large.headerlink,.rst-content table>caption .btn .fa-large.headerlink,.rst-content table>caption .nav .fa-large.headerlink,.rst-content tt.download .btn span.fa-large:first-child,.rst-content tt.download .nav span.fa-large:first-child,.wy-menu-vertical li .btn button.fa-large.toctree-expand,.wy-menu-vertical li .nav button.fa-large.toctree-expand{line-height:.9em}.btn .fa-spin.icon,.btn .fa.fa-spin,.btn .rst-content .code-block-caption .fa-spin.headerlink,.btn .rst-content .eqno .fa-spin.headerlink,.btn .rst-content .fa-spin.admonition-title,.btn .rst-content code.download span.fa-spin:first-child,.btn .rst-content dl dt .fa-spin.headerlink,.btn .rst-content h1 .fa-spin.headerlink,.btn .rst-content h2 .fa-spin.headerlink,.btn .rst-content h3 .fa-spin.headerlink,.btn .rst-content h4 .fa-spin.headerlink,.btn .rst-content h5 .fa-spin.headerlink,.btn .rst-content h6 .fa-spin.headerlink,.btn .rst-content p .fa-spin.headerlink,.btn .rst-content table>caption .fa-spin.headerlink,.btn .rst-content tt.download span.fa-spin:first-child,.btn .wy-menu-vertical li button.fa-spin.toctree-expand,.nav .fa-spin.icon,.nav .fa.fa-spin,.nav .rst-content .code-block-caption .fa-spin.headerlink,.nav .rst-content .eqno .fa-spin.headerlink,.nav .rst-content .fa-spin.admonition-title,.nav .rst-content code.download span.fa-spin:first-child,.nav .rst-content dl dt .fa-spin.headerlink,.nav .rst-content h1 .fa-spin.headerlink,.nav .rst-content h2 .fa-spin.headerlink,.nav .rst-content h3 .fa-spin.headerlink,.nav .rst-content h4 .fa-spin.headerlink,.nav .rst-content h5 .fa-spin.headerlink,.nav .rst-content h6 .fa-spin.headerlink,.nav .rst-content p .fa-spin.headerlink,.nav .rst-content table>caption .fa-spin.headerlink,.nav .rst-content tt.download span.fa-spin:first-child,.nav .wy-menu-vertical li button.fa-spin.toctree-expand,.rst-content .btn .fa-spin.admonition-title,.rst-content .code-block-caption .btn .fa-spin.headerlink,.rst-content .code-block-caption .nav .fa-spin.headerlink,.rst-content .eqno .btn .fa-spin.headerlink,.rst-content .eqno .nav .fa-spin.headerlink,.rst-content .nav .fa-spin.admonition-title,.rst-content code.download .btn span.fa-spin:first-child,.rst-content code.download .nav span.fa-spin:first-child,.rst-content dl dt .btn .fa-spin.headerlink,.rst-content dl dt .nav .fa-spin.headerlink,.rst-content h1 .btn .fa-spin.headerlink,.rst-content h1 .nav .fa-spin.headerlink,.rst-content h2 .btn .fa-spin.headerlink,.rst-content h2 .nav .fa-spin.headerlink,.rst-content h3 .btn .fa-spin.headerlink,.rst-content h3 .nav .fa-spin.headerlink,.rst-content h4 .btn .fa-spin.headerlink,.rst-content h4 .nav .fa-spin.headerlink,.rst-content h5 .btn .fa-spin.headerlink,.rst-content h5 .nav .fa-spin.headerlink,.rst-content h6 .btn .fa-spin.headerlink,.rst-content h6 .nav .fa-spin.headerlink,.rst-content p .btn .fa-spin.headerlink,.rst-content p .nav .fa-spin.headerlink,.rst-content table>caption .btn .fa-spin.headerlink,.rst-content table>caption .nav .fa-spin.headerlink,.rst-content tt.download .btn span.fa-spin:first-child,.rst-content tt.download .nav span.fa-spin:first-child,.wy-menu-vertical li .btn button.fa-spin.toctree-expand,.wy-menu-vertical li .nav button.fa-spin.toctree-expand{display:inline-block}.btn.fa:before,.btn.icon:before,.rst-content .btn.admonition-title:before,.rst-content .code-block-caption .btn.headerlink:before,.rst-content .eqno .btn.headerlink:before,.rst-content code.download span.btn:first-child:before,.rst-content dl dt .btn.headerlink:before,.rst-content h1 .btn.headerlink:before,.rst-content h2 .btn.headerlink:before,.rst-content h3 .btn.headerlink:before,.rst-content h4 .btn.headerlink:before,.rst-content h5 .btn.headerlink:before,.rst-content h6 .btn.headerlink:before,.rst-content p .btn.headerlink:before,.rst-content table>caption .btn.headerlink:before,.rst-content tt.download span.btn:first-child:before,.wy-menu-vertical li button.btn.toctree-expand:before{opacity:.5;-webkit-transition:opacity .05s ease-in;-moz-transition:opacity .05s ease-in;transition:opacity .05s ease-in}.btn.fa:hover:before,.btn.icon:hover:before,.rst-content .btn.admonition-title:hover:before,.rst-content .code-block-caption .btn.headerlink:hover:before,.rst-content .eqno .btn.headerlink:hover:before,.rst-content code.download span.btn:first-child:hover:before,.rst-content dl dt .btn.headerlink:hover:before,.rst-content h1 .btn.headerlink:hover:before,.rst-content h2 .btn.headerlink:hover:before,.rst-content h3 .btn.headerlink:hover:before,.rst-content h4 .btn.headerlink:hover:before,.rst-content h5 .btn.headerlink:hover:before,.rst-content h6 .btn.headerlink:hover:before,.rst-content p .btn.headerlink:hover:before,.rst-content table>caption .btn.headerlink:hover:before,.rst-content tt.download span.btn:first-child:hover:before,.wy-menu-vertical li button.btn.toctree-expand:hover:before{opacity:1}.btn-mini .fa:before,.btn-mini .icon:before,.btn-mini .rst-content .admonition-title:before,.btn-mini .rst-content .code-block-caption .headerlink:before,.btn-mini .rst-content .eqno .headerlink:before,.btn-mini .rst-content code.download span:first-child:before,.btn-mini .rst-content dl dt .headerlink:before,.btn-mini .rst-content h1 .headerlink:before,.btn-mini .rst-content h2 .headerlink:before,.btn-mini .rst-content h3 .headerlink:before,.btn-mini .rst-content h4 .headerlink:before,.btn-mini .rst-content h5 .headerlink:before,.btn-mini .rst-content h6 .headerlink:before,.btn-mini .rst-content p .headerlink:before,.btn-mini .rst-content table>caption .headerlink:before,.btn-mini .rst-content tt.download span:first-child:before,.btn-mini .wy-menu-vertical li button.toctree-expand:before,.rst-content .btn-mini .admonition-title:before,.rst-content .code-block-caption .btn-mini .headerlink:before,.rst-content .eqno .btn-mini .headerlink:before,.rst-content code.download .btn-mini span:first-child:before,.rst-content dl dt .btn-mini .headerlink:before,.rst-content h1 .btn-mini .headerlink:before,.rst-content h2 .btn-mini .headerlink:before,.rst-content h3 .btn-mini .headerlink:before,.rst-content h4 .btn-mini .headerlink:before,.rst-content h5 .btn-mini .headerlink:before,.rst-content h6 .btn-mini .headerlink:before,.rst-content p .btn-mini .headerlink:before,.rst-content table>caption .btn-mini .headerlink:before,.rst-content tt.download .btn-mini span:first-child:before,.wy-menu-vertical li .btn-mini button.toctree-expand:before{font-size:14px;vertical-align:-15%}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.wy-alert{padding:12px;line-height:24px;margin-bottom:24px;background:#e7f2fa}.rst-content .admonition-title,.wy-alert-title{font-weight:700;display:block;color:#fff;background:#6ab0de;padding:6px 12px;margin:-12px -12px 12px}.rst-content .danger,.rst-content .error,.rst-content .wy-alert-danger.admonition,.rst-content .wy-alert-danger.admonition-todo,.rst-content .wy-alert-danger.attention,.rst-content .wy-alert-danger.caution,.rst-content .wy-alert-danger.hint,.rst-content .wy-alert-danger.important,.rst-content .wy-alert-danger.note,.rst-content .wy-alert-danger.seealso,.rst-content .wy-alert-danger.tip,.rst-content .wy-alert-danger.warning,.wy-alert.wy-alert-danger{background:#fdf3f2}.rst-content .danger .admonition-title,.rst-content .danger .wy-alert-title,.rst-content .error .admonition-title,.rst-content .error .wy-alert-title,.rst-content .wy-alert-danger.admonition-todo .admonition-title,.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,.rst-content .wy-alert-danger.admonition .admonition-title,.rst-content .wy-alert-danger.admonition .wy-alert-title,.rst-content .wy-alert-danger.attention .admonition-title,.rst-content .wy-alert-danger.attention .wy-alert-title,.rst-content .wy-alert-danger.caution .admonition-title,.rst-content .wy-alert-danger.caution .wy-alert-title,.rst-content .wy-alert-danger.hint .admonition-title,.rst-content .wy-alert-danger.hint .wy-alert-title,.rst-content .wy-alert-danger.important .admonition-title,.rst-content .wy-alert-danger.important .wy-alert-title,.rst-content .wy-alert-danger.note .admonition-title,.rst-content .wy-alert-danger.note .wy-alert-title,.rst-content .wy-alert-danger.seealso .admonition-title,.rst-content .wy-alert-danger.seealso .wy-alert-title,.rst-content .wy-alert-danger.tip .admonition-title,.rst-content .wy-alert-danger.tip .wy-alert-title,.rst-content .wy-alert-danger.warning .admonition-title,.rst-content .wy-alert-danger.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-danger .admonition-title,.wy-alert.wy-alert-danger .rst-content .admonition-title,.wy-alert.wy-alert-danger .wy-alert-title{background:#f29f97}.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .warning,.rst-content .wy-alert-warning.admonition,.rst-content .wy-alert-warning.danger,.rst-content .wy-alert-warning.error,.rst-content .wy-alert-warning.hint,.rst-content .wy-alert-warning.important,.rst-content .wy-alert-warning.note,.rst-content .wy-alert-warning.seealso,.rst-content .wy-alert-warning.tip,.wy-alert.wy-alert-warning{background:#ffedcc}.rst-content .admonition-todo .admonition-title,.rst-content .admonition-todo .wy-alert-title,.rst-content .attention .admonition-title,.rst-content .attention .wy-alert-title,.rst-content .caution .admonition-title,.rst-content .caution .wy-alert-title,.rst-content .warning .admonition-title,.rst-content .warning .wy-alert-title,.rst-content .wy-alert-warning.admonition .admonition-title,.rst-content .wy-alert-warning.admonition .wy-alert-title,.rst-content .wy-alert-warning.danger .admonition-title,.rst-content .wy-alert-warning.danger .wy-alert-title,.rst-content .wy-alert-warning.error .admonition-title,.rst-content .wy-alert-warning.error .wy-alert-title,.rst-content .wy-alert-warning.hint .admonition-title,.rst-content .wy-alert-warning.hint .wy-alert-title,.rst-content .wy-alert-warning.important .admonition-title,.rst-content .wy-alert-warning.important .wy-alert-title,.rst-content .wy-alert-warning.note .admonition-title,.rst-content .wy-alert-warning.note .wy-alert-title,.rst-content .wy-alert-warning.seealso .admonition-title,.rst-content .wy-alert-warning.seealso .wy-alert-title,.rst-content .wy-alert-warning.tip .admonition-title,.rst-content .wy-alert-warning.tip .wy-alert-title,.rst-content .wy-alert.wy-alert-warning .admonition-title,.wy-alert.wy-alert-warning .rst-content .admonition-title,.wy-alert.wy-alert-warning .wy-alert-title{background:#f0b37e}.rst-content .note,.rst-content .seealso,.rst-content .wy-alert-info.admonition,.rst-content .wy-alert-info.admonition-todo,.rst-content .wy-alert-info.attention,.rst-content .wy-alert-info.caution,.rst-content .wy-alert-info.danger,.rst-content .wy-alert-info.error,.rst-content .wy-alert-info.hint,.rst-content .wy-alert-info.important,.rst-content .wy-alert-info.tip,.rst-content .wy-alert-info.warning,.wy-alert.wy-alert-info{background:#e7f2fa}.rst-content .note .admonition-title,.rst-content .note .wy-alert-title,.rst-content .seealso .admonition-title,.rst-content .seealso .wy-alert-title,.rst-content .wy-alert-info.admonition-todo .admonition-title,.rst-content .wy-alert-info.admonition-todo .wy-alert-title,.rst-content .wy-alert-info.admonition .admonition-title,.rst-content .wy-alert-info.admonition .wy-alert-title,.rst-content .wy-alert-info.attention .admonition-title,.rst-content .wy-alert-info.attention .wy-alert-title,.rst-content .wy-alert-info.caution .admonition-title,.rst-content .wy-alert-info.caution .wy-alert-title,.rst-content .wy-alert-info.danger .admonition-title,.rst-content .wy-alert-info.danger .wy-alert-title,.rst-content .wy-alert-info.error .admonition-title,.rst-content .wy-alert-info.error .wy-alert-title,.rst-content .wy-alert-info.hint .admonition-title,.rst-content .wy-alert-info.hint .wy-alert-title,.rst-content .wy-alert-info.important .admonition-title,.rst-content .wy-alert-info.important .wy-alert-title,.rst-content .wy-alert-info.tip .admonition-title,.rst-content .wy-alert-info.tip .wy-alert-title,.rst-content .wy-alert-info.warning .admonition-title,.rst-content .wy-alert-info.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-info .admonition-title,.wy-alert.wy-alert-info .rst-content .admonition-title,.wy-alert.wy-alert-info .wy-alert-title{background:#6ab0de}.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .wy-alert-success.admonition,.rst-content .wy-alert-success.admonition-todo,.rst-content .wy-alert-success.attention,.rst-content .wy-alert-success.caution,.rst-content .wy-alert-success.danger,.rst-content .wy-alert-success.error,.rst-content .wy-alert-success.note,.rst-content .wy-alert-success.seealso,.rst-content .wy-alert-success.warning,.wy-alert.wy-alert-success{background:#dbfaf4}.rst-content .hint .admonition-title,.rst-content .hint .wy-alert-title,.rst-content .important .admonition-title,.rst-content .important .wy-alert-title,.rst-content .tip .admonition-title,.rst-content .tip .wy-alert-title,.rst-content .wy-alert-success.admonition-todo .admonition-title,.rst-content .wy-alert-success.admonition-todo .wy-alert-title,.rst-content .wy-alert-success.admonition .admonition-title,.rst-content .wy-alert-success.admonition .wy-alert-title,.rst-content .wy-alert-success.attention .admonition-title,.rst-content .wy-alert-success.attention .wy-alert-title,.rst-content .wy-alert-success.caution .admonition-title,.rst-content .wy-alert-success.caution .wy-alert-title,.rst-content .wy-alert-success.danger .admonition-title,.rst-content .wy-alert-success.danger .wy-alert-title,.rst-content .wy-alert-success.error .admonition-title,.rst-content .wy-alert-success.error .wy-alert-title,.rst-content .wy-alert-success.note .admonition-title,.rst-content .wy-alert-success.note .wy-alert-title,.rst-content .wy-alert-success.seealso .admonition-title,.rst-content .wy-alert-success.seealso .wy-alert-title,.rst-content .wy-alert-success.warning .admonition-title,.rst-content .wy-alert-success.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-success .admonition-title,.wy-alert.wy-alert-success .rst-content .admonition-title,.wy-alert.wy-alert-success .wy-alert-title{background:#1abc9c}.rst-content .wy-alert-neutral.admonition,.rst-content .wy-alert-neutral.admonition-todo,.rst-content .wy-alert-neutral.attention,.rst-content .wy-alert-neutral.caution,.rst-content .wy-alert-neutral.danger,.rst-content .wy-alert-neutral.error,.rst-content .wy-alert-neutral.hint,.rst-content .wy-alert-neutral.important,.rst-content .wy-alert-neutral.note,.rst-content .wy-alert-neutral.seealso,.rst-content .wy-alert-neutral.tip,.rst-content .wy-alert-neutral.warning,.wy-alert.wy-alert-neutral{background:#f3f6f6}.rst-content .wy-alert-neutral.admonition-todo .admonition-title,.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,.rst-content .wy-alert-neutral.admonition .admonition-title,.rst-content .wy-alert-neutral.admonition .wy-alert-title,.rst-content .wy-alert-neutral.attention .admonition-title,.rst-content .wy-alert-neutral.attention .wy-alert-title,.rst-content .wy-alert-neutral.caution .admonition-title,.rst-content .wy-alert-neutral.caution .wy-alert-title,.rst-content .wy-alert-neutral.danger .admonition-title,.rst-content .wy-alert-neutral.danger .wy-alert-title,.rst-content .wy-alert-neutral.error .admonition-title,.rst-content .wy-alert-neutral.error .wy-alert-title,.rst-content .wy-alert-neutral.hint .admonition-title,.rst-content .wy-alert-neutral.hint .wy-alert-title,.rst-content .wy-alert-neutral.important .admonition-title,.rst-content .wy-alert-neutral.important .wy-alert-title,.rst-content .wy-alert-neutral.note .admonition-title,.rst-content .wy-alert-neutral.note .wy-alert-title,.rst-content .wy-alert-neutral.seealso .admonition-title,.rst-content .wy-alert-neutral.seealso .wy-alert-title,.rst-content .wy-alert-neutral.tip .admonition-title,.rst-content .wy-alert-neutral.tip .wy-alert-title,.rst-content .wy-alert-neutral.warning .admonition-title,.rst-content .wy-alert-neutral.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-neutral .admonition-title,.wy-alert.wy-alert-neutral .rst-content .admonition-title,.wy-alert.wy-alert-neutral .wy-alert-title{color:#404040;background:#e1e4e5}.rst-content .wy-alert-neutral.admonition-todo a,.rst-content .wy-alert-neutral.admonition a,.rst-content .wy-alert-neutral.attention a,.rst-content .wy-alert-neutral.caution a,.rst-content .wy-alert-neutral.danger a,.rst-content .wy-alert-neutral.error a,.rst-content .wy-alert-neutral.hint a,.rst-content .wy-alert-neutral.important a,.rst-content .wy-alert-neutral.note a,.rst-content .wy-alert-neutral.seealso a,.rst-content .wy-alert-neutral.tip a,.rst-content .wy-alert-neutral.warning a,.wy-alert.wy-alert-neutral a{color:#2980b9}.rst-content .admonition-todo p:last-child,.rst-content .admonition p:last-child,.rst-content .attention p:last-child,.rst-content .caution p:last-child,.rst-content .danger p:last-child,.rst-content .error p:last-child,.rst-content .hint p:last-child,.rst-content .important p:last-child,.rst-content .note p:last-child,.rst-content .seealso p:last-child,.rst-content .tip p:last-child,.rst-content .warning p:last-child,.wy-alert p:last-child{margin-bottom:0}.wy-tray-container{position:fixed;bottom:0;left:0;z-index:600}.wy-tray-container li{display:block;width:300px;background:transparent;color:#fff;text-align:center;box-shadow:0 5px 5px 0 rgba(0,0,0,.1);padding:0 24px;min-width:20%;opacity:0;height:0;line-height:56px;overflow:hidden;-webkit-transition:all .3s ease-in;-moz-transition:all .3s ease-in;transition:all .3s ease-in}.wy-tray-container li.wy-tray-item-success{background:#27ae60}.wy-tray-container li.wy-tray-item-info{background:#2980b9}.wy-tray-container li.wy-tray-item-warning{background:#e67e22}.wy-tray-container li.wy-tray-item-danger{background:#e74c3c}.wy-tray-container li.on{opacity:1;height:56px}@media screen and (max-width:768px){.wy-tray-container{bottom:auto;top:0;width:100%}.wy-tray-container li{width:100%}}button{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle;cursor:pointer;line-height:normal;-webkit-appearance:button;*overflow:visible}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}button[disabled]{cursor:default}.btn{display:inline-block;border-radius:2px;line-height:normal;white-space:nowrap;text-align:center;cursor:pointer;font-size:100%;padding:6px 12px 8px;color:#fff;border:1px solid rgba(0,0,0,.1);background-color:#27ae60;text-decoration:none;font-weight:400;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 2px -1px hsla(0,0%,100%,.5),inset 0 -2px 0 0 rgba(0,0,0,.1);outline-none:false;vertical-align:middle;*display:inline;zoom:1;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:all .1s linear;-moz-transition:all .1s linear;transition:all .1s linear}.btn-hover{background:#2e8ece;color:#fff}.btn:hover{background:#2cc36b;color:#fff}.btn:focus{background:#2cc36b;outline:0}.btn:active{box-shadow:inset 0 -1px 0 0 rgba(0,0,0,.05),inset 0 2px 0 0 rgba(0,0,0,.1);padding:8px 12px 6px}.btn:visited{color:#fff}.btn-disabled,.btn-disabled:active,.btn-disabled:focus,.btn-disabled:hover,.btn:disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:.4;cursor:not-allowed;box-shadow:none}.btn::-moz-focus-inner{padding:0;border:0}.btn-small{font-size:80%}.btn-info{background-color:#2980b9!important}.btn-info:hover{background-color:#2e8ece!important}.btn-neutral{background-color:#f3f6f6!important;color:#404040!important}.btn-neutral:hover{background-color:#e5ebeb!important;color:#404040}.btn-neutral:visited{color:#404040!important}.btn-success{background-color:#27ae60!important}.btn-success:hover{background-color:#295!important}.btn-danger{background-color:#e74c3c!important}.btn-danger:hover{background-color:#ea6153!important}.btn-warning{background-color:#e67e22!important}.btn-warning:hover{background-color:#e98b39!important}.btn-invert{background-color:#222}.btn-invert:hover{background-color:#2f2f2f!important}.btn-link{background-color:transparent!important;color:#2980b9;box-shadow:none;border-color:transparent!important}.btn-link:active,.btn-link:hover{background-color:transparent!important;color:#409ad5!important;box-shadow:none}.btn-link:visited{color:#9b59b6}.wy-btn-group .btn,.wy-control .btn{vertical-align:middle}.wy-btn-group{margin-bottom:24px;*zoom:1}.wy-btn-group:after,.wy-btn-group:before{display:table;content:""}.wy-btn-group:after{clear:both}.wy-dropdown{position:relative;display:inline-block}.wy-dropdown-active .wy-dropdown-menu{display:block}.wy-dropdown-menu{position:absolute;left:0;display:none;float:left;top:100%;min-width:100%;background:#fcfcfc;z-index:100;border:1px solid #cfd7dd;box-shadow:0 2px 2px 0 rgba(0,0,0,.1);padding:12px}.wy-dropdown-menu>dd>a{display:block;clear:both;color:#404040;white-space:nowrap;font-size:90%;padding:0 12px;cursor:pointer}.wy-dropdown-menu>dd>a:hover{background:#2980b9;color:#fff}.wy-dropdown-menu>dd.divider{border-top:1px solid #cfd7dd;margin:6px 0}.wy-dropdown-menu>dd.search{padding-bottom:12px}.wy-dropdown-menu>dd.search input[type=search]{width:100%}.wy-dropdown-menu>dd.call-to-action{background:#e3e3e3;text-transform:uppercase;font-weight:500;font-size:80%}.wy-dropdown-menu>dd.call-to-action:hover{background:#e3e3e3}.wy-dropdown-menu>dd.call-to-action .btn{color:#fff}.wy-dropdown.wy-dropdown-up .wy-dropdown-menu{bottom:100%;top:auto;left:auto;right:0}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu{background:#fcfcfc;margin-top:2px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a{padding:6px 12px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover{background:#2980b9;color:#fff}.wy-dropdown.wy-dropdown-left .wy-dropdown-menu{right:0;left:auto;text-align:right}.wy-dropdown-arrow:before{content:" ";border-bottom:5px solid #f5f5f5;border-left:5px solid transparent;border-right:5px solid transparent;position:absolute;display:block;top:-4px;left:50%;margin-left:-3px}.wy-dropdown-arrow.wy-dropdown-arrow-left:before{left:11px}.wy-form-stacked select{display:block}.wy-form-aligned .wy-help-inline,.wy-form-aligned input,.wy-form-aligned label,.wy-form-aligned select,.wy-form-aligned textarea{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-form-aligned .wy-control-group>label{display:inline-block;vertical-align:middle;width:10em;margin:6px 12px 0 0;float:left}.wy-form-aligned .wy-control{float:left}.wy-form-aligned .wy-control label{display:block}.wy-form-aligned .wy-control select{margin-top:6px}fieldset{margin:0}fieldset,legend{border:0;padding:0}legend{width:100%;white-space:normal;margin-bottom:24px;font-size:150%;*margin-left:-7px}label,legend{display:block}label{margin:0 0 .3125em;color:#333;font-size:90%}input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}.wy-control-group{margin-bottom:24px;max-width:1200px;margin-left:auto;margin-right:auto;*zoom:1}.wy-control-group:after,.wy-control-group:before{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group.wy-control-group-required>label:after{content:" *";color:#e74c3c}.wy-control-group .wy-form-full,.wy-control-group .wy-form-halves,.wy-control-group .wy-form-thirds{padding-bottom:12px}.wy-control-group .wy-form-full input[type=color],.wy-control-group .wy-form-full input[type=date],.wy-control-group .wy-form-full input[type=datetime-local],.wy-control-group .wy-form-full input[type=datetime],.wy-control-group .wy-form-full input[type=email],.wy-control-group .wy-form-full input[type=month],.wy-control-group .wy-form-full input[type=number],.wy-control-group .wy-form-full input[type=password],.wy-control-group .wy-form-full input[type=search],.wy-control-group .wy-form-full input[type=tel],.wy-control-group .wy-form-full input[type=text],.wy-control-group .wy-form-full input[type=time],.wy-control-group .wy-form-full input[type=url],.wy-control-group .wy-form-full input[type=week],.wy-control-group .wy-form-full select,.wy-control-group .wy-form-halves input[type=color],.wy-control-group .wy-form-halves input[type=date],.wy-control-group .wy-form-halves input[type=datetime-local],.wy-control-group .wy-form-halves input[type=datetime],.wy-control-group .wy-form-halves input[type=email],.wy-control-group .wy-form-halves input[type=month],.wy-control-group .wy-form-halves input[type=number],.wy-control-group .wy-form-halves input[type=password],.wy-control-group .wy-form-halves input[type=search],.wy-control-group .wy-form-halves input[type=tel],.wy-control-group .wy-form-halves input[type=text],.wy-control-group .wy-form-halves input[type=time],.wy-control-group .wy-form-halves input[type=url],.wy-control-group .wy-form-halves input[type=week],.wy-control-group .wy-form-halves select,.wy-control-group .wy-form-thirds input[type=color],.wy-control-group .wy-form-thirds input[type=date],.wy-control-group .wy-form-thirds input[type=datetime-local],.wy-control-group .wy-form-thirds input[type=datetime],.wy-control-group .wy-form-thirds input[type=email],.wy-control-group .wy-form-thirds input[type=month],.wy-control-group .wy-form-thirds input[type=number],.wy-control-group .wy-form-thirds input[type=password],.wy-control-group .wy-form-thirds input[type=search],.wy-control-group .wy-form-thirds input[type=tel],.wy-control-group .wy-form-thirds input[type=text],.wy-control-group .wy-form-thirds input[type=time],.wy-control-group .wy-form-thirds input[type=url],.wy-control-group .wy-form-thirds input[type=week],.wy-control-group .wy-form-thirds select{width:100%}.wy-control-group .wy-form-full{float:left;display:block;width:100%;margin-right:0}.wy-control-group .wy-form-full:last-child{margin-right:0}.wy-control-group .wy-form-halves{float:left;display:block;margin-right:2.35765%;width:48.82117%}.wy-control-group .wy-form-halves:last-child,.wy-control-group .wy-form-halves:nth-of-type(2n){margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(odd){clear:left}.wy-control-group .wy-form-thirds{float:left;display:block;margin-right:2.35765%;width:31.76157%}.wy-control-group .wy-form-thirds:last-child,.wy-control-group .wy-form-thirds:nth-of-type(3n){margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n+1){clear:left}.wy-control-group.wy-control-group-no-input .wy-control,.wy-control-no-input{margin:6px 0 0;font-size:90%}.wy-control-no-input{display:inline-block}.wy-control-group.fluid-input input[type=color],.wy-control-group.fluid-input input[type=date],.wy-control-group.fluid-input input[type=datetime-local],.wy-control-group.fluid-input input[type=datetime],.wy-control-group.fluid-input input[type=email],.wy-control-group.fluid-input input[type=month],.wy-control-group.fluid-input input[type=number],.wy-control-group.fluid-input input[type=password],.wy-control-group.fluid-input input[type=search],.wy-control-group.fluid-input input[type=tel],.wy-control-group.fluid-input input[type=text],.wy-control-group.fluid-input input[type=time],.wy-control-group.fluid-input input[type=url],.wy-control-group.fluid-input input[type=week]{width:100%}.wy-form-message-inline{padding-left:.3em;color:#666;font-size:90%}.wy-form-message{display:block;color:#999;font-size:70%;margin-top:.3125em;font-style:italic}.wy-form-message p{font-size:inherit;font-style:italic;margin-bottom:6px}.wy-form-message p:last-child{margin-bottom:0}input{line-height:normal}input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;*overflow:visible}input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week]{-webkit-appearance:none;padding:6px;display:inline-block;border:1px solid #ccc;font-size:80%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 3px #ddd;border-radius:0;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}input[type=datetime-local]{padding:.34375em .625em}input[disabled]{cursor:default}input[type=checkbox],input[type=radio]{padding:0;margin-right:.3125em;*height:13px;*width:13px}input[type=checkbox],input[type=radio],input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}input[type=color]:focus,input[type=date]:focus,input[type=datetime-local]:focus,input[type=datetime]:focus,input[type=email]:focus,input[type=month]:focus,input[type=number]:focus,input[type=password]:focus,input[type=search]:focus,input[type=tel]:focus,input[type=text]:focus,input[type=time]:focus,input[type=url]:focus,input[type=week]:focus{outline:0;outline:thin dotted\9;border-color:#333}input.no-focus:focus{border-color:#ccc!important}input[type=checkbox]:focus,input[type=file]:focus,input[type=radio]:focus{outline:thin dotted #333;outline:1px auto #129fea}input[type=color][disabled],input[type=date][disabled],input[type=datetime-local][disabled],input[type=datetime][disabled],input[type=email][disabled],input[type=month][disabled],input[type=number][disabled],input[type=password][disabled],input[type=search][disabled],input[type=tel][disabled],input[type=text][disabled],input[type=time][disabled],input[type=url][disabled],input[type=week][disabled]{cursor:not-allowed;background-color:#fafafa}input:focus:invalid,select:focus:invalid,textarea:focus:invalid{color:#e74c3c;border:1px solid #e74c3c}input:focus:invalid:focus,select:focus:invalid:focus,textarea:focus:invalid:focus{border-color:#e74c3c}input[type=checkbox]:focus:invalid:focus,input[type=file]:focus:invalid:focus,input[type=radio]:focus:invalid:focus{outline-color:#e74c3c}input.wy-input-large{padding:12px;font-size:100%}textarea{overflow:auto;vertical-align:top;width:100%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif}select,textarea{padding:.5em .625em;display:inline-block;border:1px solid #ccc;font-size:80%;box-shadow:inset 0 1px 3px #ddd;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}select{border:1px solid #ccc;background-color:#fff}select[multiple]{height:auto}select:focus,textarea:focus{outline:0}input[readonly],select[disabled],select[readonly],textarea[disabled],textarea[readonly]{cursor:not-allowed;background-color:#fafafa}input[type=checkbox][disabled],input[type=radio][disabled]{cursor:not-allowed}.wy-checkbox,.wy-radio{margin:6px 0;color:#404040;display:block}.wy-checkbox input,.wy-radio input{vertical-align:baseline}.wy-form-message-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-input-prefix,.wy-input-suffix{white-space:nowrap;padding:6px}.wy-input-prefix .wy-input-context,.wy-input-suffix .wy-input-context{line-height:27px;padding:0 8px;display:inline-block;font-size:80%;background-color:#f3f6f6;border:1px solid #ccc;color:#999}.wy-input-suffix .wy-input-context{border-left:0}.wy-input-prefix .wy-input-context{border-right:0}.wy-switch{position:relative;display:block;height:24px;margin-top:12px;cursor:pointer}.wy-switch:before{left:0;top:0;width:36px;height:12px;background:#ccc}.wy-switch:after,.wy-switch:before{position:absolute;content:"";display:block;border-radius:4px;-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.wy-switch:after{width:18px;height:18px;background:#999;left:-3px;top:-3px}.wy-switch span{position:absolute;left:48px;display:block;font-size:12px;color:#ccc;line-height:1}.wy-switch.active:before{background:#1e8449}.wy-switch.active:after{left:24px;background:#27ae60}.wy-switch.disabled{cursor:not-allowed;opacity:.8}.wy-control-group.wy-control-group-error .wy-form-message,.wy-control-group.wy-control-group-error>label{color:#e74c3c}.wy-control-group.wy-control-group-error input[type=color],.wy-control-group.wy-control-group-error input[type=date],.wy-control-group.wy-control-group-error input[type=datetime-local],.wy-control-group.wy-control-group-error input[type=datetime],.wy-control-group.wy-control-group-error input[type=email],.wy-control-group.wy-control-group-error input[type=month],.wy-control-group.wy-control-group-error input[type=number],.wy-control-group.wy-control-group-error input[type=password],.wy-control-group.wy-control-group-error input[type=search],.wy-control-group.wy-control-group-error input[type=tel],.wy-control-group.wy-control-group-error input[type=text],.wy-control-group.wy-control-group-error input[type=time],.wy-control-group.wy-control-group-error input[type=url],.wy-control-group.wy-control-group-error input[type=week],.wy-control-group.wy-control-group-error textarea{border:1px solid #e74c3c}.wy-inline-validate{white-space:nowrap}.wy-inline-validate .wy-input-context{padding:.5em .625em;display:inline-block;font-size:80%}.wy-inline-validate.wy-inline-validate-success .wy-input-context{color:#27ae60}.wy-inline-validate.wy-inline-validate-danger .wy-input-context{color:#e74c3c}.wy-inline-validate.wy-inline-validate-warning .wy-input-context{color:#e67e22}.wy-inline-validate.wy-inline-validate-info .wy-input-context{color:#2980b9}.rotate-90{-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.rotate-180{-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.rotate-270{-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.mirror{-webkit-transform:scaleX(-1);-moz-transform:scaleX(-1);-ms-transform:scaleX(-1);-o-transform:scaleX(-1);transform:scaleX(-1)}.mirror.rotate-90{-webkit-transform:scaleX(-1) rotate(90deg);-moz-transform:scaleX(-1) rotate(90deg);-ms-transform:scaleX(-1) rotate(90deg);-o-transform:scaleX(-1) rotate(90deg);transform:scaleX(-1) rotate(90deg)}.mirror.rotate-180{-webkit-transform:scaleX(-1) rotate(180deg);-moz-transform:scaleX(-1) rotate(180deg);-ms-transform:scaleX(-1) rotate(180deg);-o-transform:scaleX(-1) rotate(180deg);transform:scaleX(-1) rotate(180deg)}.mirror.rotate-270{-webkit-transform:scaleX(-1) rotate(270deg);-moz-transform:scaleX(-1) rotate(270deg);-ms-transform:scaleX(-1) rotate(270deg);-o-transform:scaleX(-1) rotate(270deg);transform:scaleX(-1) rotate(270deg)}@media only screen and (max-width:480px){.wy-form button[type=submit]{margin:.7em 0 0}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=text],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week],.wy-form label{margin-bottom:.3em;display:block}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week]{margin-bottom:0}.wy-form-aligned .wy-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.wy-form-aligned .wy-control{margin:1.5em 0 0}.wy-form-message,.wy-form-message-inline,.wy-form .wy-help-inline{display:block;font-size:80%;padding:6px 0}}@media screen and (max-width:768px){.tablet-hide{display:none}}@media screen and (max-width:480px){.mobile-hide{display:none}}.float-left{float:left}.float-right{float:right}.full-width{width:100%}.rst-content table.docutils,.rst-content table.field-list,.wy-table{border-collapse:collapse;border-spacing:0;empty-cells:show;margin-bottom:24px}.rst-content table.docutils caption,.rst-content table.field-list caption,.wy-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.rst-content table.docutils td,.rst-content table.docutils th,.rst-content table.field-list td,.rst-content table.field-list th,.wy-table td,.wy-table th{font-size:90%;margin:0;overflow:visible;padding:8px 16px}.rst-content table.docutils td:first-child,.rst-content table.docutils th:first-child,.rst-content table.field-list td:first-child,.rst-content table.field-list th:first-child,.wy-table td:first-child,.wy-table th:first-child{border-left-width:0}.rst-content table.docutils thead,.rst-content table.field-list thead,.wy-table thead{color:#000;text-align:left;vertical-align:bottom;white-space:nowrap}.rst-content table.docutils thead th,.rst-content table.field-list thead th,.wy-table thead th{font-weight:700;border-bottom:2px solid #e1e4e5}.rst-content table.docutils td,.rst-content table.field-list td,.wy-table td{background-color:transparent;vertical-align:middle}.rst-content table.docutils td p,.rst-content table.field-list td p,.wy-table td p{line-height:18px}.rst-content table.docutils td p:last-child,.rst-content table.field-list td p:last-child,.wy-table td p:last-child{margin-bottom:0}.rst-content table.docutils .wy-table-cell-min,.rst-content table.field-list .wy-table-cell-min,.wy-table .wy-table-cell-min{width:1%;padding-right:0}.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox],.wy-table .wy-table-cell-min input[type=checkbox]{margin:0}.wy-table-secondary{color:grey;font-size:90%}.wy-table-tertiary{color:grey;font-size:80%}.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,.wy-table-backed,.wy-table-odd td,.wy-table-striped tr:nth-child(2n-1) td{background-color:#f3f6f6}.rst-content table.docutils,.wy-table-bordered-all{border:1px solid #e1e4e5}.rst-content table.docutils td,.wy-table-bordered-all td{border-bottom:1px solid #e1e4e5;border-left:1px solid #e1e4e5}.rst-content table.docutils tbody>tr:last-child td,.wy-table-bordered-all tbody>tr:last-child td{border-bottom-width:0}.wy-table-bordered{border:1px solid #e1e4e5}.wy-table-bordered-rows td{border-bottom:1px solid #e1e4e5}.wy-table-bordered-rows tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal td,.wy-table-horizontal th{border-width:0 0 1px;border-bottom:1px solid #e1e4e5}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-responsive{margin-bottom:24px;max-width:100%;overflow:auto}.wy-table-responsive table{margin-bottom:0!important}.wy-table-responsive table td,.wy-table-responsive table th{white-space:nowrap}a{color:#2980b9;text-decoration:none;cursor:pointer}a:hover{color:#3091d1}a:visited{color:#9b59b6}html{height:100%}body,html{overflow-x:hidden}body{font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-weight:400;color:#404040;min-height:100%;background:#edf0f2}.wy-text-left{text-align:left}.wy-text-center{text-align:center}.wy-text-right{text-align:right}.wy-text-large{font-size:120%}.wy-text-normal{font-size:100%}.wy-text-small,small{font-size:80%}.wy-text-strike{text-decoration:line-through}.wy-text-warning{color:#e67e22!important}a.wy-text-warning:hover{color:#eb9950!important}.wy-text-info{color:#2980b9!important}a.wy-text-info:hover{color:#409ad5!important}.wy-text-success{color:#27ae60!important}a.wy-text-success:hover{color:#36d278!important}.wy-text-danger{color:#e74c3c!important}a.wy-text-danger:hover{color:#ed7669!important}.wy-text-neutral{color:#404040!important}a.wy-text-neutral:hover{color:#595959!important}.rst-content .toctree-wrapper>p.caption,h1,h2,h3,h4,h5,h6,legend{margin-top:0;font-weight:700;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif}p{line-height:24px;font-size:16px;margin:0 0 24px}h1{font-size:175%}.rst-content .toctree-wrapper>p.caption,h2{font-size:150%}h3{font-size:125%}h4{font-size:115%}h5{font-size:110%}h6{font-size:100%}hr{display:block;height:1px;border:0;border-top:1px solid #e1e4e5;margin:24px 0;padding:0}.rst-content code,.rst-content tt,code{white-space:nowrap;max-width:100%;background:#fff;border:1px solid #e1e4e5;font-size:75%;padding:0 5px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#e74c3c;overflow-x:auto}.rst-content tt.code-large,code.code-large{font-size:90%}.rst-content .section ul,.rst-content .toctree-wrapper ul,.rst-content section ul,.wy-plain-list-disc,article ul{list-style:disc;line-height:24px;margin-bottom:24px}.rst-content .section ul li,.rst-content .toctree-wrapper ul li,.rst-content section ul li,.wy-plain-list-disc li,article ul li{list-style:disc;margin-left:24px}.rst-content .section ul li p:last-child,.rst-content .section ul li ul,.rst-content .toctree-wrapper ul li p:last-child,.rst-content .toctree-wrapper ul li ul,.rst-content section ul li p:last-child,.rst-content section ul li ul,.wy-plain-list-disc li p:last-child,.wy-plain-list-disc li ul,article ul li p:last-child,article ul li ul{margin-bottom:0}.rst-content .section ul li li,.rst-content .toctree-wrapper ul li li,.rst-content section ul li li,.wy-plain-list-disc li li,article ul li li{list-style:circle}.rst-content .section ul li li li,.rst-content .toctree-wrapper ul li li li,.rst-content section ul li li li,.wy-plain-list-disc li li li,article ul li li li{list-style:square}.rst-content .section ul li ol li,.rst-content .toctree-wrapper ul li ol li,.rst-content section ul li ol li,.wy-plain-list-disc li ol li,article ul li ol li{list-style:decimal}.rst-content .section ol,.rst-content .section ol.arabic,.rst-content .toctree-wrapper ol,.rst-content .toctree-wrapper ol.arabic,.rst-content section ol,.rst-content section ol.arabic,.wy-plain-list-decimal,article ol{list-style:decimal;line-height:24px;margin-bottom:24px}.rst-content .section ol.arabic li,.rst-content .section ol li,.rst-content .toctree-wrapper ol.arabic li,.rst-content .toctree-wrapper ol li,.rst-content section ol.arabic li,.rst-content section ol li,.wy-plain-list-decimal li,article ol li{list-style:decimal;margin-left:24px}.rst-content .section ol.arabic li ul,.rst-content .section ol li p:last-child,.rst-content .section ol li ul,.rst-content .toctree-wrapper ol.arabic li ul,.rst-content .toctree-wrapper ol li p:last-child,.rst-content .toctree-wrapper ol li ul,.rst-content section ol.arabic li ul,.rst-content section ol li p:last-child,.rst-content section ol li ul,.wy-plain-list-decimal li p:last-child,.wy-plain-list-decimal li ul,article ol li p:last-child,article ol li ul{margin-bottom:0}.rst-content .section ol.arabic li ul li,.rst-content .section ol li ul li,.rst-content .toctree-wrapper ol.arabic li ul li,.rst-content .toctree-wrapper ol li ul li,.rst-content section ol.arabic li ul li,.rst-content section ol li ul li,.wy-plain-list-decimal li ul li,article ol li ul li{list-style:disc}.wy-breadcrumbs{*zoom:1}.wy-breadcrumbs:after,.wy-breadcrumbs:before{display:table;content:""}.wy-breadcrumbs:after{clear:both}.wy-breadcrumbs>li{display:inline-block;padding-top:5px}.wy-breadcrumbs>li.wy-breadcrumbs-aside{float:right}.rst-content .wy-breadcrumbs>li code,.rst-content .wy-breadcrumbs>li tt,.wy-breadcrumbs>li .rst-content tt,.wy-breadcrumbs>li code{all:inherit;color:inherit}.breadcrumb-item:before{content:"/";color:#bbb;font-size:13px;padding:0 6px 0 3px}.wy-breadcrumbs-extra{margin-bottom:0;color:#b3b3b3;font-size:80%;display:inline-block}@media screen and (max-width:480px){.wy-breadcrumbs-extra,.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}@media print{.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}html{font-size:16px}.wy-affix{position:fixed;top:1.618em}.wy-menu a:hover{text-decoration:none}.wy-menu-horiz{*zoom:1}.wy-menu-horiz:after,.wy-menu-horiz:before{display:table;content:""}.wy-menu-horiz:after{clear:both}.wy-menu-horiz li,.wy-menu-horiz ul{display:inline-block}.wy-menu-horiz li:hover{background:hsla(0,0%,100%,.1)}.wy-menu-horiz li.divide-left{border-left:1px solid #404040}.wy-menu-horiz li.divide-right{border-right:1px solid #404040}.wy-menu-horiz a{height:32px;display:inline-block;line-height:32px;padding:0 16px}.wy-menu-vertical{width:300px}.wy-menu-vertical header,.wy-menu-vertical p.caption{color:#55a5d9;height:32px;line-height:32px;padding:0 1.618em;margin:12px 0 0;display:block;font-weight:700;text-transform:uppercase;font-size:85%;white-space:nowrap}.wy-menu-vertical ul{margin-bottom:0}.wy-menu-vertical li.divide-top{border-top:1px solid #404040}.wy-menu-vertical li.divide-bottom{border-bottom:1px solid #404040}.wy-menu-vertical li.current{background:#e3e3e3}.wy-menu-vertical li.current a{color:grey;border-right:1px solid #c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.current a:hover{background:#d6d6d6}.rst-content .wy-menu-vertical li tt,.wy-menu-vertical li .rst-content tt,.wy-menu-vertical li code{border:none;background:inherit;color:inherit;padding-left:0;padding-right:0}.wy-menu-vertical li button.toctree-expand{display:block;float:left;margin-left:-1.2em;line-height:18px;color:#4d4d4d;border:none;background:none;padding:0}.wy-menu-vertical li.current>a,.wy-menu-vertical li.on a{color:#404040;font-weight:700;position:relative;background:#fcfcfc;border:none;padding:.4045em 1.618em}.wy-menu-vertical li.current>a:hover,.wy-menu-vertical li.on a:hover{background:#fcfcfc}.wy-menu-vertical li.current>a:hover button.toctree-expand,.wy-menu-vertical li.on a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand{display:block;line-height:18px;color:#333}.wy-menu-vertical li.toctree-l1.current>a{border-bottom:1px solid #c9c9c9;border-top:1px solid #c9c9c9}.wy-menu-vertical .toctree-l1.current .toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .toctree-l11>ul{display:none}.wy-menu-vertical .toctree-l1.current .current.toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .current.toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .current.toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .current.toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .current.toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .current.toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .current.toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .current.toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .current.toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .current.toctree-l11>ul{display:block}.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4{font-size:.9em}.wy-menu-vertical li.toctree-l2 a,.wy-menu-vertical li.toctree-l3 a,.wy-menu-vertical li.toctree-l4 a,.wy-menu-vertical li.toctree-l5 a,.wy-menu-vertical li.toctree-l6 a,.wy-menu-vertical li.toctree-l7 a,.wy-menu-vertical li.toctree-l8 a,.wy-menu-vertical li.toctree-l9 a,.wy-menu-vertical li.toctree-l10 a{color:#404040}.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l4 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l5 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l6 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l7 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l8 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l9 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l10 a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a,.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a,.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a,.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a,.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a,.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a,.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a,.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{display:block}.wy-menu-vertical li.toctree-l2.current>a{padding:.4045em 2.427em}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{padding:.4045em 1.618em .4045em 4.045em}.wy-menu-vertical li.toctree-l3.current>a{padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{padding:.4045em 1.618em .4045em 5.663em}.wy-menu-vertical li.toctree-l4.current>a{padding:.4045em 5.663em}.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a{padding:.4045em 1.618em .4045em 7.281em}.wy-menu-vertical li.toctree-l5.current>a{padding:.4045em 7.281em}.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a{padding:.4045em 1.618em .4045em 8.899em}.wy-menu-vertical li.toctree-l6.current>a{padding:.4045em 8.899em}.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a{padding:.4045em 1.618em .4045em 10.517em}.wy-menu-vertical li.toctree-l7.current>a{padding:.4045em 10.517em}.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a{padding:.4045em 1.618em .4045em 12.135em}.wy-menu-vertical li.toctree-l8.current>a{padding:.4045em 12.135em}.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a{padding:.4045em 1.618em .4045em 13.753em}.wy-menu-vertical li.toctree-l9.current>a{padding:.4045em 13.753em}.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a{padding:.4045em 1.618em .4045em 15.371em}.wy-menu-vertical li.toctree-l10.current>a{padding:.4045em 15.371em}.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{padding:.4045em 1.618em .4045em 16.989em}.wy-menu-vertical li.toctree-l2.current>a,.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{background:#c9c9c9}.wy-menu-vertical li.toctree-l2 button.toctree-expand{color:#a3a3a3}.wy-menu-vertical li.toctree-l3.current>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{background:#bdbdbd}.wy-menu-vertical li.toctree-l3 button.toctree-expand{color:#969696}.wy-menu-vertical li.current ul{display:block}.wy-menu-vertical li ul{margin-bottom:0;display:none}.wy-menu-vertical li ul li a{margin-bottom:0;color:#d9d9d9;font-weight:400}.wy-menu-vertical a{line-height:18px;padding:.4045em 1.618em;display:block;position:relative;font-size:90%;color:#d9d9d9}.wy-menu-vertical a:hover{background-color:#4e4a4a;cursor:pointer}.wy-menu-vertical a:hover button.toctree-expand{color:#d9d9d9}.wy-menu-vertical a:active{background-color:#2980b9;cursor:pointer;color:#fff}.wy-menu-vertical a:active button.toctree-expand{color:#fff}.wy-side-nav-search{display:block;width:300px;padding:.809em;margin-bottom:.809em;z-index:200;background-color:#2980b9;text-align:center;color:#fcfcfc}.wy-side-nav-search input[type=text]{width:100%;border-radius:50px;padding:6px 12px;border-color:#2472a4}.wy-side-nav-search img{display:block;margin:auto auto .809em;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-side-nav-search .wy-dropdown>a,.wy-side-nav-search>a{color:#fcfcfc;font-size:100%;font-weight:700;display:inline-block;padding:4px 6px;margin-bottom:.809em;max-width:100%}.wy-side-nav-search .wy-dropdown>a:hover,.wy-side-nav-search .wy-dropdown>aactive,.wy-side-nav-search .wy-dropdown>afocus,.wy-side-nav-search>a:hover,.wy-side-nav-search>aactive,.wy-side-nav-search>afocus{background:hsla(0,0%,100%,.1)}.wy-side-nav-search .wy-dropdown>a img.logo,.wy-side-nav-search>a img.logo{display:block;margin:0 auto;height:auto;width:auto;border-radius:0;max-width:100%;background:transparent}.wy-side-nav-search .wy-dropdown>a.icon,.wy-side-nav-search>a.icon{display:block}.wy-side-nav-search .wy-dropdown>a.icon img.logo,.wy-side-nav-search>a.icon img.logo{margin-top:.85em}.wy-side-nav-search>div.switch-menus{position:relative;display:block;margin-top:-.4045em;margin-bottom:.809em;font-weight:400;color:hsla(0,0%,100%,.3)}.wy-side-nav-search>div.switch-menus>div.language-switch,.wy-side-nav-search>div.switch-menus>div.version-switch{display:inline-block;padding:.2em}.wy-side-nav-search>div.switch-menus>div.language-switch select,.wy-side-nav-search>div.switch-menus>div.version-switch select{display:inline-block;margin-right:-2rem;padding-right:2rem;max-width:240px;text-align-last:center;background:none;border:none;border-radius:0;box-shadow:none;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-size:1em;font-weight:400;color:hsla(0,0%,100%,.3);cursor:pointer;appearance:none;-webkit-appearance:none;-moz-appearance:none}.wy-side-nav-search>div.switch-menus>div.language-switch select:active,.wy-side-nav-search>div.switch-menus>div.language-switch select:focus,.wy-side-nav-search>div.switch-menus>div.language-switch select:hover,.wy-side-nav-search>div.switch-menus>div.version-switch select:active,.wy-side-nav-search>div.switch-menus>div.version-switch select:focus,.wy-side-nav-search>div.switch-menus>div.version-switch select:hover{background:hsla(0,0%,100%,.1);color:hsla(0,0%,100%,.5)}.wy-side-nav-search>div.switch-menus>div.language-switch select option,.wy-side-nav-search>div.switch-menus>div.version-switch select option{color:#000}.wy-side-nav-search>div.switch-menus>div.language-switch:has(>select):after,.wy-side-nav-search>div.switch-menus>div.version-switch:has(>select):after{display:inline-block;width:1.5em;height:100%;padding:.1em;content:"\f0d7";font-size:1em;line-height:1.2em;font-family:FontAwesome;text-align:center;pointer-events:none;box-sizing:border-box}.wy-nav .wy-menu-vertical header{color:#2980b9}.wy-nav .wy-menu-vertical a{color:#b3b3b3}.wy-nav .wy-menu-vertical a:hover{background-color:#2980b9;color:#fff}[data-menu-wrap]{-webkit-transition:all .2s ease-in;-moz-transition:all .2s ease-in;transition:all .2s ease-in;position:absolute;opacity:1;width:100%;opacity:0}[data-menu-wrap].move-center{left:0;right:auto;opacity:1}[data-menu-wrap].move-left{right:auto;left:-100%;opacity:0}[data-menu-wrap].move-right{right:-100%;left:auto;opacity:0}.wy-body-for-nav{background:#fcfcfc}.wy-grid-for-nav{position:absolute;width:100%;height:100%}.wy-nav-side{position:fixed;top:0;bottom:0;left:0;padding-bottom:2em;width:300px;overflow-x:hidden;overflow-y:hidden;min-height:100%;color:#9b9b9b;background:#343131;z-index:200}.wy-side-scroll{width:320px;position:relative;overflow-x:hidden;overflow-y:scroll;height:100%}.wy-nav-top{display:none;background:#2980b9;color:#fff;padding:.4045em .809em;position:relative;line-height:50px;text-align:center;font-size:100%;*zoom:1}.wy-nav-top:after,.wy-nav-top:before{display:table;content:""}.wy-nav-top:after{clear:both}.wy-nav-top a{color:#fff;font-weight:700}.wy-nav-top img{margin-right:12px;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-nav-top i{font-size:30px;float:left;cursor:pointer;padding-top:inherit}.wy-nav-content-wrap{margin-left:300px;background:#fcfcfc;min-height:100%}.wy-nav-content{padding:1.618em 3.236em;height:100%;max-width:800px;margin:auto}.wy-body-mask{position:fixed;width:100%;height:100%;background:rgba(0,0,0,.2);display:none;z-index:499}.wy-body-mask.on{display:block}footer{color:grey}footer p{margin-bottom:12px}.rst-content footer span.commit tt,footer span.commit .rst-content tt,footer span.commit code{padding:0;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:1em;background:none;border:none;color:grey}.rst-footer-buttons{*zoom:1}.rst-footer-buttons:after,.rst-footer-buttons:before{width:100%;display:table;content:""}.rst-footer-buttons:after{clear:both}.rst-breadcrumbs-buttons{margin-top:12px;*zoom:1}.rst-breadcrumbs-buttons:after,.rst-breadcrumbs-buttons:before{display:table;content:""}.rst-breadcrumbs-buttons:after{clear:both}#search-results .search li{margin-bottom:24px;border-bottom:1px solid #e1e4e5;padding-bottom:24px}#search-results .search li:first-child{border-top:1px solid #e1e4e5;padding-top:24px}#search-results .search li a{font-size:120%;margin-bottom:12px;display:inline-block}#search-results .context{color:grey;font-size:90%}.genindextable li>ul{margin-left:24px}@media screen and (max-width:768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}.wy-nav-side{left:-300px}.wy-nav-side.shift{width:85%;left:0}.wy-menu.wy-menu-vertical,.wy-side-nav-search,.wy-side-scroll{width:auto}.wy-nav-content-wrap{margin-left:0}.wy-nav-content-wrap .wy-nav-content{padding:1.618em}.wy-nav-content-wrap.shift{position:fixed;min-width:100%;left:85%;top:0;height:100%;overflow:hidden}}@media screen and (min-width:1100px){.wy-nav-content-wrap{background:rgba(0,0,0,.05)}.wy-nav-content{margin:0;background:#fcfcfc}}@media print{.rst-versions,.wy-nav-side,footer{display:none}.wy-nav-content-wrap{margin-left:0}}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60;*zoom:1}.rst-versions .rst-current-version:after,.rst-versions .rst-current-version:before{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-content .code-block-caption .rst-versions .rst-current-version .headerlink,.rst-content .eqno .rst-versions .rst-current-version .headerlink,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-content code.download .rst-versions .rst-current-version span:first-child,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-content p .rst-versions .rst-current-version .headerlink,.rst-content table>caption .rst-versions .rst-current-version .headerlink,.rst-content tt.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .icon,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-versions .rst-current-version .rst-content .code-block-caption .headerlink,.rst-versions .rst-current-version .rst-content .eqno .headerlink,.rst-versions .rst-current-version .rst-content code.download span:first-child,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-versions .rst-current-version .rst-content p .headerlink,.rst-versions .rst-current-version .rst-content table>caption .headerlink,.rst-versions .rst-current-version .rst-content tt.download span:first-child,.rst-versions .rst-current-version .wy-menu-vertical li button.toctree-expand,.wy-menu-vertical li .rst-versions .rst-current-version button.toctree-expand{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}.rst-content .toctree-wrapper>p.caption,.rst-content h1,.rst-content h2,.rst-content h3,.rst-content h4,.rst-content h5,.rst-content h6{margin-bottom:24px}.rst-content img{max-width:100%;height:auto}.rst-content div.figure,.rst-content figure{margin-bottom:24px}.rst-content div.figure .caption-text,.rst-content figure .caption-text{font-style:italic}.rst-content div.figure p:last-child.caption,.rst-content figure p:last-child.caption{margin-bottom:0}.rst-content div.figure.align-center,.rst-content figure.align-center{text-align:center}.rst-content .section>a>img,.rst-content .section>img,.rst-content section>a>img,.rst-content section>img{margin-bottom:24px}.rst-content abbr[title]{text-decoration:none}.rst-content.style-external-links a.reference.external:after{font-family:FontAwesome;content:"\f08e";color:#b3b3b3;vertical-align:super;font-size:60%;margin:0 .2em}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content pre.literal-block{white-space:pre;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;display:block;overflow:auto}.rst-content div[class^=highlight],.rst-content pre.literal-block{border:1px solid #e1e4e5;overflow-x:auto;margin:1px 0 24px}.rst-content div[class^=highlight] div[class^=highlight],.rst-content pre.literal-block div[class^=highlight]{padding:0;border:none;margin:0}.rst-content div[class^=highlight] td.code{width:100%}.rst-content .linenodiv pre{border-right:1px solid #e6e9ea;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;user-select:none;pointer-events:none}.rst-content div[class^=highlight] pre{white-space:pre;margin:0;padding:12px;display:block;overflow:auto}.rst-content div[class^=highlight] pre .hll{display:block;margin:0 -12px;padding:0 12px}.rst-content .linenodiv pre,.rst-content div[class^=highlight] pre,.rst-content pre.literal-block{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:12px;line-height:1.4}.rst-content div.highlight .gp,.rst-content div.highlight span.linenos{user-select:none;pointer-events:none}.rst-content div.highlight span.linenos{display:inline-block;padding-left:0;padding-right:12px;margin-right:12px;border-right:1px solid #e6e9ea}.rst-content .code-block-caption{font-style:italic;font-size:85%;line-height:1;padding:1em 0;text-align:center}@media print{.rst-content .codeblock,.rst-content div[class^=highlight],.rst-content div[class^=highlight] pre{white-space:pre-wrap}}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning{clear:both}.rst-content .admonition-todo .last,.rst-content .admonition-todo>:last-child,.rst-content .admonition .last,.rst-content .admonition>:last-child,.rst-content .attention .last,.rst-content .attention>:last-child,.rst-content .caution .last,.rst-content .caution>:last-child,.rst-content .danger .last,.rst-content .danger>:last-child,.rst-content .error .last,.rst-content .error>:last-child,.rst-content .hint .last,.rst-content .hint>:last-child,.rst-content .important .last,.rst-content .important>:last-child,.rst-content .note .last,.rst-content .note>:last-child,.rst-content .seealso .last,.rst-content .seealso>:last-child,.rst-content .tip .last,.rst-content .tip>:last-child,.rst-content .warning .last,.rst-content .warning>:last-child{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent!important;border-color:rgba(0,0,0,.1)!important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha>li,.rst-content .toctree-wrapper ol.loweralpha,.rst-content .toctree-wrapper ol.loweralpha>li,.rst-content section ol.loweralpha,.rst-content section ol.loweralpha>li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha>li,.rst-content .toctree-wrapper ol.upperalpha,.rst-content .toctree-wrapper ol.upperalpha>li,.rst-content section ol.upperalpha,.rst-content section ol.upperalpha>li{list-style:upper-alpha}.rst-content .section ol li>*,.rst-content .section ul li>*,.rst-content .toctree-wrapper ol li>*,.rst-content .toctree-wrapper ul li>*,.rst-content section ol li>*,.rst-content section ul li>*{margin-top:12px;margin-bottom:12px}.rst-content .section ol li>:first-child,.rst-content .section ul li>:first-child,.rst-content .toctree-wrapper ol li>:first-child,.rst-content .toctree-wrapper ul li>:first-child,.rst-content section ol li>:first-child,.rst-content section ul li>:first-child{margin-top:0}.rst-content .section ol li>p,.rst-content .section ol li>p:last-child,.rst-content .section ul li>p,.rst-content .section ul li>p:last-child,.rst-content .toctree-wrapper ol li>p,.rst-content .toctree-wrapper ol li>p:last-child,.rst-content .toctree-wrapper ul li>p,.rst-content .toctree-wrapper ul li>p:last-child,.rst-content section ol li>p,.rst-content section ol li>p:last-child,.rst-content section ul li>p,.rst-content section ul li>p:last-child{margin-bottom:12px}.rst-content .section ol li>p:only-child,.rst-content .section ol li>p:only-child:last-child,.rst-content .section ul li>p:only-child,.rst-content .section ul li>p:only-child:last-child,.rst-content .toctree-wrapper ol li>p:only-child,.rst-content .toctree-wrapper ol li>p:only-child:last-child,.rst-content .toctree-wrapper ul li>p:only-child,.rst-content .toctree-wrapper ul li>p:only-child:last-child,.rst-content section ol li>p:only-child,.rst-content section ol li>p:only-child:last-child,.rst-content section ul li>p:only-child,.rst-content section ul li>p:only-child:last-child{margin-bottom:0}.rst-content .section ol li>ol,.rst-content .section ol li>ul,.rst-content .section ul li>ol,.rst-content .section ul li>ul,.rst-content .toctree-wrapper ol li>ol,.rst-content .toctree-wrapper ol li>ul,.rst-content .toctree-wrapper ul li>ol,.rst-content .toctree-wrapper ul li>ul,.rst-content section ol li>ol,.rst-content section ol li>ul,.rst-content section ul li>ol,.rst-content section ul li>ul{margin-bottom:12px}.rst-content .section ol.simple li>*,.rst-content .section ol.simple li ol,.rst-content .section ol.simple li ul,.rst-content .section ul.simple li>*,.rst-content .section ul.simple li ol,.rst-content .section ul.simple li ul,.rst-content .toctree-wrapper ol.simple li>*,.rst-content .toctree-wrapper ol.simple li ol,.rst-content .toctree-wrapper ol.simple li ul,.rst-content .toctree-wrapper ul.simple li>*,.rst-content .toctree-wrapper ul.simple li ol,.rst-content .toctree-wrapper ul.simple li ul,.rst-content section ol.simple li>*,.rst-content section ol.simple li ol,.rst-content section ol.simple li ul,.rst-content section ul.simple li>*,.rst-content section ul.simple li ol,.rst-content section ul.simple li ul{margin-top:0;margin-bottom:0}.rst-content .line-block{margin-left:0;margin-bottom:24px;line-height:24px}.rst-content .line-block .line-block{margin-left:24px;margin-bottom:0}.rst-content .topic-title{font-weight:700;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0 0 24px 24px}.rst-content .align-left{float:left;margin:0 24px 24px 0}.rst-content .align-center{margin:auto}.rst-content .align-center:not(table){display:block}.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink{opacity:0;font-size:14px;font-family:FontAwesome;margin-left:.5em}.rst-content .code-block-caption .headerlink:focus,.rst-content .code-block-caption:hover .headerlink,.rst-content .eqno .headerlink:focus,.rst-content .eqno:hover .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink:focus,.rst-content .toctree-wrapper>p.caption:hover .headerlink,.rst-content dl dt .headerlink:focus,.rst-content dl dt:hover .headerlink,.rst-content h1 .headerlink:focus,.rst-content h1:hover .headerlink,.rst-content h2 .headerlink:focus,.rst-content h2:hover .headerlink,.rst-content h3 .headerlink:focus,.rst-content h3:hover .headerlink,.rst-content h4 .headerlink:focus,.rst-content h4:hover .headerlink,.rst-content h5 .headerlink:focus,.rst-content h5:hover .headerlink,.rst-content h6 .headerlink:focus,.rst-content h6:hover .headerlink,.rst-content p.caption .headerlink:focus,.rst-content p.caption:hover .headerlink,.rst-content p .headerlink:focus,.rst-content p:hover .headerlink,.rst-content table>caption .headerlink:focus,.rst-content table>caption:hover .headerlink{opacity:1}.rst-content p a{overflow-wrap:anywhere}.rst-content .wy-table td p,.rst-content .wy-table td ul,.rst-content .wy-table th p,.rst-content .wy-table th ul,.rst-content table.docutils td p,.rst-content table.docutils td ul,.rst-content table.docutils th p,.rst-content table.docutils th ul,.rst-content table.field-list td p,.rst-content table.field-list td ul,.rst-content table.field-list th p,.rst-content table.field-list th ul{font-size:inherit}.rst-content .btn:focus{outline:2px solid}.rst-content table>caption .headerlink:after{font-size:12px}.rst-content .centered{text-align:center}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:1px solid #e1e4e5}.rst-content .sidebar dl,.rst-content .sidebar p,.rst-content .sidebar ul{font-size:90%}.rst-content .sidebar .last,.rst-content .sidebar>:last-child{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif;font-weight:700;background:#e1e4e5;padding:6px 12px;margin:-24px -24px 24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;box-shadow:0 0 0 2px #f1c40f;display:inline;font-weight:700}.rst-content .citation-reference,.rst-content .footnote-reference{vertical-align:baseline;position:relative;top:-.4em;line-height:0;font-size:90%}.rst-content .citation-reference>span.fn-bracket,.rst-content .footnote-reference>span.fn-bracket{display:none}.rst-content .hlist{width:100%}.rst-content dl dt span.classifier:before{content:" : "}.rst-content dl dt span.classifier-delimiter{display:none!important}html.writer-html4 .rst-content table.docutils.citation,html.writer-html4 .rst-content table.docutils.footnote{background:none;border:none}html.writer-html4 .rst-content table.docutils.citation td,html.writer-html4 .rst-content table.docutils.citation tr,html.writer-html4 .rst-content table.docutils.footnote td,html.writer-html4 .rst-content table.docutils.footnote tr{border:none;background-color:transparent!important;white-space:normal}html.writer-html4 .rst-content table.docutils.citation td.label,html.writer-html4 .rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{display:grid;grid-template-columns:auto minmax(80%,95%)}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{display:inline-grid;grid-template-columns:max-content auto}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{display:grid;grid-template-columns:auto auto minmax(.65rem,auto) minmax(40%,95%)}html.writer-html5 .rst-content aside.citation>span.label,html.writer-html5 .rst-content aside.footnote>span.label,html.writer-html5 .rst-content div.citation>span.label{grid-column-start:1;grid-column-end:2}html.writer-html5 .rst-content aside.citation>span.backrefs,html.writer-html5 .rst-content aside.footnote>span.backrefs,html.writer-html5 .rst-content div.citation>span.backrefs{grid-column-start:2;grid-column-end:3;grid-row-start:1;grid-row-end:3}html.writer-html5 .rst-content aside.citation>p,html.writer-html5 .rst-content aside.footnote>p,html.writer-html5 .rst-content div.citation>p{grid-column-start:4;grid-column-end:5}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{margin-bottom:24px}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{padding-left:1rem}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dd,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dd,html.writer-html5 .rst-content dl.footnote>dt{margin-bottom:0}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{font-size:.9rem}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.footnote>dt{margin:0 .5rem .5rem 0;line-height:1.2rem;word-break:break-all;font-weight:400}html.writer-html5 .rst-content dl.citation>dt>span.brackets:before,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:before{content:"["}html.writer-html5 .rst-content dl.citation>dt>span.brackets:after,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:after{content:"]"}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a{word-break:keep-all}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a:not(:first-child):before,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.footnote>dd{margin:0 0 .5rem;line-height:1.2rem}html.writer-html5 .rst-content dl.citation>dd p,html.writer-html5 .rst-content dl.footnote>dd p{font-size:.9rem}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{padding-left:1rem;padding-right:1rem;font-size:.9rem;line-height:1.2rem}html.writer-html5 .rst-content aside.citation p,html.writer-html5 .rst-content aside.footnote p,html.writer-html5 .rst-content div.citation p{font-size:.9rem;line-height:1.2rem;margin-bottom:12px}html.writer-html5 .rst-content aside.citation span.backrefs,html.writer-html5 .rst-content aside.footnote span.backrefs,html.writer-html5 .rst-content div.citation span.backrefs{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content aside.citation span.backrefs>a,html.writer-html5 .rst-content aside.footnote span.backrefs>a,html.writer-html5 .rst-content div.citation span.backrefs>a{word-break:keep-all}html.writer-html5 .rst-content aside.citation span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content aside.footnote span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content div.citation span.backrefs>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content aside.citation span.label,html.writer-html5 .rst-content aside.footnote span.label,html.writer-html5 .rst-content div.citation span.label{line-height:1.2rem}html.writer-html5 .rst-content aside.citation-list,html.writer-html5 .rst-content aside.footnote-list,html.writer-html5 .rst-content div.citation-list{margin-bottom:24px}html.writer-html5 .rst-content dl.option-list kbd{font-size:.9rem}.rst-content table.docutils.footnote,html.writer-html4 .rst-content table.docutils.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content aside.footnote-list aside.footnote,html.writer-html5 .rst-content div.citation-list>div.citation,html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{color:grey}.rst-content table.docutils.footnote code,.rst-content table.docutils.footnote tt,html.writer-html4 .rst-content table.docutils.citation code,html.writer-html4 .rst-content table.docutils.citation tt,html.writer-html5 .rst-content aside.footnote-list aside.footnote code,html.writer-html5 .rst-content aside.footnote-list aside.footnote tt,html.writer-html5 .rst-content aside.footnote code,html.writer-html5 .rst-content aside.footnote tt,html.writer-html5 .rst-content div.citation-list>div.citation code,html.writer-html5 .rst-content div.citation-list>div.citation tt,html.writer-html5 .rst-content dl.citation code,html.writer-html5 .rst-content dl.citation tt,html.writer-html5 .rst-content dl.footnote code,html.writer-html5 .rst-content dl.footnote tt{color:#555}.rst-content .wy-table-responsive.citation,.rst-content .wy-table-responsive.footnote{margin-bottom:0}.rst-content .wy-table-responsive.citation+:not(.citation),.rst-content .wy-table-responsive.footnote+:not(.footnote){margin-top:24px}.rst-content .wy-table-responsive.citation:last-child,.rst-content .wy-table-responsive.footnote:last-child{margin-bottom:24px}.rst-content table.docutils th{border-color:#e1e4e5}html.writer-html5 .rst-content table.docutils th{border:1px solid #e1e4e5}html.writer-html5 .rst-content table.docutils td>p,html.writer-html5 .rst-content table.docutils th>p{line-height:1rem;margin-bottom:0;font-size:.9rem}.rst-content table.docutils td .last,.rst-content table.docutils td .last>:last-child{margin-bottom:0}.rst-content table.field-list,.rst-content table.field-list td{border:none}.rst-content table.field-list td p{line-height:inherit}.rst-content table.field-list td>strong{display:inline-block}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left}.rst-content code,.rst-content tt{color:#000;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;padding:2px 5px}.rst-content code big,.rst-content code em,.rst-content tt big,.rst-content tt em{font-size:100%!important;line-height:normal}.rst-content code.literal,.rst-content tt.literal{color:#e74c3c;white-space:normal}.rst-content code.xref,.rst-content tt.xref,a .rst-content code,a .rst-content tt{font-weight:700;color:#404040;overflow-wrap:normal}.rst-content kbd,.rst-content pre,.rst-content samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace}.rst-content a code,.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:700;margin-bottom:12px}.rst-content dl ol,.rst-content dl p,.rst-content dl table,.rst-content dl ul{margin-bottom:12px}.rst-content dl dd{margin:0 0 12px 24px;line-height:24px}.rst-content dl dd>ol:last-child,.rst-content dl dd>p:last-child,.rst-content dl dd>table:last-child,.rst-content dl dd>ul:last-child{margin-bottom:0}html.writer-html4 .rst-content dl:not(.docutils),html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple){margin-bottom:24px}html.writer-html4 .rst-content dl:not(.docutils)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{display:table;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:3px solid #6ab0de;padding:6px;position:relative}html.writer-html4 .rst-content dl:not(.docutils)>dt:before,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:before{color:#6ab0de}html.writer-html4 .rst-content dl:not(.docutils)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{margin-bottom:6px;border:none;border-left:3px solid #ccc;background:#f0f0f0;color:#555}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils)>dt:first-child,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:first-child{margin-top:0}html.writer-html4 .rst-content dl:not(.docutils) code.descclassname,html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descclassname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{background-color:transparent;border:none;padding:0;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .optional,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .property,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .property{display:inline-block;padding-right:8px;max-width:100%}html.writer-html4 .rst-content dl:not(.docutils) .k,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .k{font-style:italic}html.writer-html4 .rst-content dl:not(.docutils) .descclassname,html.writer-html4 .rst-content dl:not(.docutils) .descname,html.writer-html4 .rst-content dl:not(.docutils) .sig-name,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .sig-name{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#000}.rst-content .viewcode-back,.rst-content .viewcode-link{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right}.rst-content p.rubric{margin-bottom:12px;font-weight:700}.rst-content code.download,.rst-content tt.download{background:inherit;padding:inherit;font-weight:400;font-family:inherit;font-size:inherit;color:inherit;border:inherit;white-space:inherit}.rst-content code.download span:first-child,.rst-content tt.download span:first-child{-webkit-font-smoothing:subpixel-antialiased}.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{margin-right:4px}.rst-content .guilabel,.rst-content .menuselection{font-size:80%;font-weight:700;border-radius:4px;padding:2.4px 6px;margin:auto 2px}.rst-content .guilabel,.rst-content .menuselection{border:1px solid #7fbbe3;background:#e7f2fa}.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>.kbd,.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>kbd{color:inherit;font-size:80%;background-color:#fff;border:1px solid #a6a6a6;border-radius:4px;box-shadow:0 2px grey;padding:2.4px 6px;margin:auto 0}.rst-content .versionmodified{font-style:italic}@media screen and (max-width:480px){.rst-content .sidebar{width:100%;float:none;margin-left:0}}span[id*=MathJax-Span]{color:#404040}.math{text-align:center}@font-face{font-family:Lato;src:url(fonts/lato-normal.woff2?bd03a2cc277bbbc338d464e679fe9942) format("woff2"),url(fonts/lato-normal.woff?27bd77b9162d388cb8d4c4217c7c5e2a) format("woff");font-weight:400;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold.woff2?cccb897485813c7c256901dbca54ecf2) format("woff2"),url(fonts/lato-bold.woff?d878b6c29b10beca227e9eef4246111b) format("woff");font-weight:700;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold-italic.woff2?0b6bb6725576b072c5d0b02ecdd1900d) format("woff2"),url(fonts/lato-bold-italic.woff?9c7e4e9eb485b4a121c760e61bc3707c) format("woff");font-weight:700;font-style:italic;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-normal-italic.woff2?4eb103b4d12be57cb1d040ed5e162e9d) format("woff2"),url(fonts/lato-normal-italic.woff?f28f2d6482446544ef1ea1ccc6dd5892) format("woff");font-weight:400;font-style:italic;font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:400;src:url(fonts/Roboto-Slab-Regular.woff2?7abf5b8d04d26a2cafea937019bca958) format("woff2"),url(fonts/Roboto-Slab-Regular.woff?c1be9284088d487c5e3ff0a10a92e58c) format("woff");font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:700;src:url(fonts/Roboto-Slab-Bold.woff2?9984f4a9bda09be08e83f2506954adbe) format("woff2"),url(fonts/Roboto-Slab-Bold.woff?bed5564a116b05148e3b3bea6fb1162a) format("woff");font-display:block}
\ No newline at end of file
diff --git a/docs/build/html/_static/sphinx_highlight.js b/docs/build/html/_static/sphinx_highlight.js
index aae669d..a74e103 100644
--- a/docs/build/html/_static/sphinx_highlight.js
+++ b/docs/build/html/_static/sphinx_highlight.js
@@ -1,7 +1,7 @@
 /* Highlighting utilities for Sphinx HTML documentation. */
 "use strict";
 
-const SPHINX_HIGHLIGHT_ENABLED = true
+const SPHINX_HIGHLIGHT_ENABLED = true;
 
 /**
  * highlight a given string on a node by wrapping it in
@@ -13,9 +13,9 @@ const _highlight = (node, addItems, text, className) => {
     const parent = node.parentNode;
     const pos = val.toLowerCase().indexOf(text);
     if (
-      pos >= 0 &&
-      !parent.classList.contains(className) &&
-      !parent.classList.contains("nohighlight")
+      pos >= 0
+      && !parent.classList.contains(className)
+      && !parent.classList.contains("nohighlight")
     ) {
       let span;
 
@@ -29,19 +29,18 @@ const _highlight = (node, addItems, text, className) => {
       }
 
       span.appendChild(document.createTextNode(val.substr(pos, text.length)));
-      parent.insertBefore(
-        span,
-        parent.insertBefore(
-          document.createTextNode(val.substr(pos + text.length)),
-          node.nextSibling
-        )
-      );
+      const rest = document.createTextNode(val.substr(pos + text.length));
+      parent.insertBefore(span, parent.insertBefore(rest, node.nextSibling));
       node.nodeValue = val.substr(0, pos);
+      /* There may be more occurrences of search term in this node. So call this
+       * function recursively on the remaining fragment.
+       */
+      _highlight(rest, addItems, text, className);
 
       if (isInSVG) {
         const rect = document.createElementNS(
           "http://www.w3.org/2000/svg",
-          "rect"
+          "rect",
         );
         const bbox = parent.getBBox();
         rect.x.baseVal.value = bbox.x;
@@ -60,7 +59,7 @@ const _highlightText = (thisNode, text, className) => {
   let addItems = [];
   _highlight(thisNode, addItems, text, className);
   addItems.forEach((obj) =>
-    obj.parent.insertAdjacentElement("beforebegin", obj.target)
+    obj.parent.insertAdjacentElement("beforebegin", obj.target),
   );
 };
 
@@ -68,25 +67,31 @@ const _highlightText = (thisNode, text, className) => {
  * Small JavaScript module for the documentation.
  */
 const SphinxHighlight = {
-
   /**
    * highlight the search words provided in localstorage in the text
    */
   highlightSearchWords: () => {
-    if (!SPHINX_HIGHLIGHT_ENABLED) return;  // bail if no highlight
+    if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight
 
     // get and clear terms from localstorage
     const url = new URL(window.location);
     const highlight =
-        localStorage.getItem("sphinx_highlight_terms")
-        || url.searchParams.get("highlight")
-        || "";
-    localStorage.removeItem("sphinx_highlight_terms")
-    url.searchParams.delete("highlight");
-    window.history.replaceState({}, "", url);
+      localStorage.getItem("sphinx_highlight_terms")
+      || url.searchParams.get("highlight")
+      || "";
+    localStorage.removeItem("sphinx_highlight_terms");
+    // Update history only if '?highlight' is present; otherwise it
+    // clears text fragments (not set in window.location by the browser)
+    if (url.searchParams.has("highlight")) {
+      url.searchParams.delete("highlight");
+      window.history.replaceState({}, "", url);
+    }
 
     // get individual terms from highlight string
-    const terms = highlight.toLowerCase().split(/\s+/).filter(x => x);
+    const terms = highlight
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((x) => x);
     if (terms.length === 0) return; // nothing to do
 
     // There should never be more than one element matching "div.body"
@@ -102,11 +107,11 @@ const SphinxHighlight = {
       document
         .createRange()
         .createContextualFragment(
-          '
' +
-            '' +
-            _("Hide Search Matches") +
-            "
"
-        )
+          '
'
+            + ''
+            + _("Hide Search Matches")
+            + "
",
+        ),
     );
   },
 
@@ -120,7 +125,7 @@ const SphinxHighlight = {
     document
       .querySelectorAll("span.highlighted")
       .forEach((el) => el.classList.remove("highlighted"));
-    localStorage.removeItem("sphinx_highlight_terms")
+    localStorage.removeItem("sphinx_highlight_terms");
   },
 
   initEscapeListener: () => {
@@ -129,10 +134,15 @@ const SphinxHighlight = {
 
     document.addEventListener("keydown", (event) => {
       // bail for input elements
-      if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return;
+      if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName))
+        return;
       // bail with special keys
-      if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return;
-      if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) {
+      if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey)
+        return;
+      if (
+        DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS
+        && event.key === "Escape"
+      ) {
         SphinxHighlight.hideSearchWords();
         event.preventDefault();
       }
@@ -140,5 +150,10 @@ const SphinxHighlight = {
   },
 };
 
-_ready(SphinxHighlight.highlightSearchWords);
-_ready(SphinxHighlight.initEscapeListener);
+_ready(() => {
+  /* Do not call highlightSearchWords() when we are on the search page.
+   * It will highlight words from the *previous* search query.
+   */
+  if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords();
+  SphinxHighlight.initEscapeListener();
+});
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 689cc6e..96270d5 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -61,7 +61,8 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
+#html_theme = 'sphinx_rtd_theme'
+html_theme = 'pydata_sphinx_theme'
 # html_theme = 'furo'
 # need to be installed: pip install furo (not working...)
 # html_static_path = ['_static']
diff --git a/quapy/error.py b/quapy/error.py
index 621d08d..f6ca125 100644
--- a/quapy/error.py
+++ b/quapy/error.py
@@ -177,6 +177,11 @@ def msre(prevs_true, prevs_hat, prevs_train, eps=0.):
 def aitchisondist(prevs_true, prevs_hat):
     """
     Computes the Aitchison distance between two prevalence vectors.
+    The Aitchison distance between prevalence vectors :math:`p` and
+    :math:`\\hat{p}` is computed as
+    :math:`d_A(p,\\hat{p})=\\|\\mathrm{clr}(p)-\\mathrm{clr}(\\hat{p})\\|_2`,
+    where :math:`\\mathrm{clr}(p)_i=\\log p_i-\\frac{1}{|\\mathcal{Y}|}
+    \\sum_{j \\in \\mathcal{Y}} \\log p_j`.
 
     :param prevs_true: array-like with the true prevalence values
     :param prevs_hat: array-like with the predicted prevalence values
@@ -191,7 +196,9 @@ def aitchisondist(prevs_true, prevs_hat):
 def maitchisondist(prevs_true, prevs_hat):
     """
     Computes the mean Aitchison distance (see :meth:`quapy.error.aitchisondist`)
-    across the sample pairs.
+    across the sample pairs, i.e.,
+    :math:`\\mathrm{mAitchisonDist}=\\frac{1}{n}\\sum_{i=1}^n
+    d_A(p_i,\\hat{p}_i)`.
 
     :param prevs_true: array-like with the true prevalence values
     :param prevs_hat: array-like with the predicted prevalence values
diff --git a/quapy/functional.py b/quapy/functional.py
index bd62377..ca47458 100644
--- a/quapy/functional.py
+++ b/quapy/functional.py
@@ -430,7 +430,7 @@ def argmin_prevalence(loss: Callable,
     :param method: string indicating the search strategy. Possible values are::
         'optim_minimize': uses scipy.optim
         'linear_search': carries out a linear search for binary problems in the space [0, 0.01, 0.02, ..., 1]
-        'ternary_search': implements the ternary search (not yet implemented)
+        'ternary_search': carries out a ternary search for binary problems in the interval [0,1]
     :return: np.ndarray, a prevalence vector
     """
     if method == 'optim_minimize':
@@ -438,7 +438,7 @@ def argmin_prevalence(loss: Callable,
     elif method == 'linear_search':
         return linear_search(loss, n_classes)
     elif method == 'ternary_search':
-        ternary_search(loss, n_classes)
+        return ternary_search(loss, n_classes)
     else:
         raise NotImplementedError()
 
@@ -493,7 +493,32 @@ def linear_search(loss: Callable, n_classes: int):
 
 
 def ternary_search(loss: Callable, n_classes: int):
-    raise NotImplementedError()
+    """
+    Performs a ternary search for the best prevalence value in binary problems.
+    This search assumes the loss is unimodal over the interval [0,1].
+
+    :param loss: (callable) the function to minimize
+    :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector
+    :return: (ndarray) the best prevalence vector found
+    """
+    assert n_classes == 2, 'ternary search is only available for binary problems'
+
+    left, right = 0., 1.
+    tol = 1e-5
+    while abs(right - left) >= tol:
+        left_third = left + (right - left) / 3
+        right_third = right - (right - left) / 3
+
+        left_loss = loss(np.asarray([1 - left_third, left_third]))
+        right_loss = loss(np.asarray([1 - right_third, right_third]))
+
+        if left_loss < right_loss:
+            right = right_third
+        else:
+            left = left_third
+
+    prev = (left + right) / 2
+    return np.asarray([1 - prev, prev])
 
 
 # ------------------------------------------------------------------------------------------
diff --git a/quapy/method/__init__.py b/quapy/method/__init__.py
index a83c42b..7f05027 100644
--- a/quapy/method/__init__.py
+++ b/quapy/method/__init__.py
@@ -15,6 +15,7 @@ AGGREGATIVE_METHODS = {
     aggregative.ACC,
     aggregative.PCC,
     aggregative.PACC,
+    aggregative.RLLS,
     aggregative.EMQ,
     aggregative.HDy,
     aggregative.DyS,
@@ -52,6 +53,7 @@ MULTICLASS_METHODS = {
     aggregative.ACC,
     aggregative.PCC,
     aggregative.PACC,
+    aggregative.RLLS,
     aggregative.EMQ,
     aggregative.KDEyML,
     aggregative.KDEyCS,
@@ -75,4 +77,3 @@ QUANTIFICATION_METHODS = AGGREGATIVE_METHODS | NON_AGGREGATIVE_METHODS | META_ME
 
 
 
-
diff --git a/quapy/method/_kdey.py b/quapy/method/_kdey.py
index 5445b34..37a6501 100644
--- a/quapy/method/_kdey.py
+++ b/quapy/method/_kdey.py
@@ -4,6 +4,7 @@ from sklearn.base import BaseEstimator
 from sklearn.neighbors import KernelDensity
 
 import quapy as qp
+from quapy.method._helper import _labels_to_indices
 from quapy.method.aggregative import AggregativeSoftQuantifier
 import quapy.functional as F
 from scipy.special import logsumexp
@@ -374,13 +375,11 @@ class KDEyCS(AggregativeSoftQuantifier):
 
         P, y = classif_predictions, labels
         n = len(self.classes_)
-
-        assert all(sorted(np.unique(y)) == np.arange(n)), \
-            'label name gaps not allowed in current implementation'
+        y = _labels_to_indices(y, self.classes_)
 
         # counts_inv keeps track of the relative weight of each datapoint within its class
         # (i.e., the weight in its KDE model)
-        counts_inv = 1 / (F.counts_from_labels(y, classes=self.classes_))
+        counts_inv = 1 / (F.counts_from_labels(y, classes=np.arange(n)))
 
         # tr_tr_sums corresponds to symbol \overline{B} in the paper
         tr_tr_sums = np.zeros(shape=(n,n), dtype=float)
diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py
index 5806b1d..5de2717 100644
--- a/quapy/method/aggregative.py
+++ b/quapy/method/aggregative.py
@@ -3,7 +3,6 @@ from argparse import ArgumentError
 from copy import deepcopy
 from typing import Callable, Literal, Union
 import numpy as np
-from numpy.f2py.crackfortran import true_intent_list
 from sklearn.base import BaseEstimator
 from sklearn.calibration import CalibratedClassifierCV
 from sklearn.exceptions import NotFittedError
@@ -17,25 +16,17 @@ from quapy.functional import get_divergence
 from quapy.classification.svmperf import SVMperf
 from quapy.data import LabelledCollection
 from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric
+from quapy.method._helper import (
+    _get_abstention_calibrators,
+    _get_cvxpy,
+    _rlls_check_mode,
+    _rlls_joint_distribution,
+    _rlls_predicted_marginal,
+    _rlls_compute_3deltaC,
+    _rlls_compute_weights,
+    _labels_to_indices,
+)
 
-# import warnings
-# from sklearn.exceptions import ConvergenceWarning
-# warnings.filterwarnings("ignore", category=ConvergenceWarning)
-
-
-def _get_abstention_calibrators():
-    try:
-        from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
-    except ImportError as exc:
-        raise ImportError(
-            "Posterior calibration for EMQ requires the optional 'abstention' package."
-        ) from exc
-    return {
-        'nbvs': NoBiasVectorScaling(),
-        'bcts': TempScaling(bias_positions='all'),
-        'ts': TempScaling(),
-        'vs': VectorScaling(),
-    }
 
 
 # Abstract classes
@@ -683,6 +674,108 @@ class PACC(AggregativeSoftQuantifier):
         return confusion.T
 
 
+class RLLS(AggregativeSoftQuantifier):
+    """
+    `Regularized Learning for Domain Adaptation under Label Shifts
+    `_, used here as an aggregative
+    quantifier.
+
+    This implementation ports the regularized weight-estimation component of
+    RLLS to QuaPy's aggregative interface. It estimates label-shift ratios from
+    validation posteriors and source labels, then rescales the source
+    prevalence to obtain target prevalence estimates.
+
+    This method relies on the optional `cvxpy` dependency.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case
+        the classifier is taken to be the one indicated in
+        `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set
+        to False if the learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier
+        predictions. This specification can be made as float in (0, 1)
+        indicating the proportion of stratified held-out validation set to be
+        extracted from the training set; or as an integer (default 5),
+        indicating that the predictions are to be generated in a `k`-fold
+        cross-validation manner; or as a tuple `(X, y)` defining the specific
+        set of data to use for validation. This method requires source
+        predictions and therefore needs `val_split` whenever
+        `fit_classifier=True`.
+    :param mode: whether source- and target-domain quantities are estimated
+        from posterior probabilities (`soft`, default) or from argmax
+        predictions (`hard`)
+    :param alpha: multiplicative factor for the regularization level (default
+        0.01)
+    :param delta: confidence parameter used in the finite-sample regularizer
+        (default 0.05)
+    :param clip_weights: if True, clips negative importance weights to zero
+        before converting them into prevalence estimates
+    :param norm: the normalization method passed to
+        :func:`quapy.functional.normalize_prevalence`
+    """
+
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier=True,
+            val_split=5,
+            mode: Literal['soft', 'hard'] = 'soft',
+            alpha: float = 0.01,
+            delta: float = 0.05,
+            clip_weights: bool = True,
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.mode = mode
+        self.alpha = alpha
+        self.delta = delta
+        self.clip_weights = clip_weights
+        self.norm = norm
+        self.last_w_ = None
+
+    def _check_init_parameters(self):
+        _get_cvxpy()
+        _rlls_check_mode(self.mode)
+        if not isinstance(self.alpha, (int, float)) or self.alpha < 0:
+            raise ValueError(f'expected a non-negative real value for alpha; found {self.alpha!r}')
+        if not isinstance(self.delta, (int, float)) or not (0 < self.delta < 1):
+            raise ValueError(f'expected delta to be in (0,1); found {self.delta!r}')
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
+        if self.fit_classifier and self.val_split is None:
+            raise ValueError(
+                'RLLS requires validation predictions for aggregation_fit; '
+                'please set val_split to an integer, float, or validation tuple'
+            )
+
+    def aggregation_fit(self, classif_predictions, labels):
+        if classif_predictions is None or labels is None:
+            raise ValueError('RLLS requires source posterior predictions and source labels')
+
+        self.train_prevalence_ = F.prevalence_from_labels(labels, classes=self.classes_)
+        self.C_zy_ = _rlls_joint_distribution(
+            classif_predictions,
+            labels,
+            self.classes_,
+            mode=self.mode,
+        )
+        self.pz_ = _rlls_predicted_marginal(classif_predictions, mode=self.mode)
+        self.rho_ = _rlls_compute_3deltaC(len(self.classes_), len(labels), self.delta)
+
+    def aggregate(self, classif_posteriors):
+        qz = _rlls_predicted_marginal(classif_posteriors, mode=self.mode)
+        w = _rlls_compute_weights(
+            self.C_zy_,
+            qz,
+            self.pz_,
+            rho=self.alpha * self.rho_,
+            clip=self.clip_weights,
+        )
+        self.last_w_ = w
+        estimate = self.train_prevalence_ * w
+        return F.normalize_prevalence(estimate, method=self.norm)
+
+
 class EMQ(AggregativeSoftQuantifier):
     """
     `Expectation Maximization for Quantification `_ (EMQ),
@@ -991,9 +1084,6 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
 
         prev_estimations = []
-        # for bins in np.linspace(10, 110, 11, dtype=int):  #[10, 20, 30, ..., 100, 110]
-        # Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)
-        # Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)
         for bins in self.bins:
             Pxy0_density = self.Pxy0_density[bins]
             Pxy1_density = self.Pxy1_density[bins]
@@ -1003,13 +1093,12 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
             # the authors proposed to search for the prevalence yielding the best matching as a linear search
             # at small steps (modern implementations resort to an optimization procedure,
             # see class DistributionMatching)
-            prev_selected, min_dist = None, None
-            for prev in F.prevalence_linspace(grid_points=101, repeats=1, smooth_limits_epsilon=0.0):
-                Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density
-                hdy = F.HellingerDistance(Px_train, Px_test)
-                if prev_selected is None or hdy < min_dist:
-                    prev_selected, min_dist = prev, hdy
-            prev_estimations.append(prev_selected)
+            def loss(prev):
+                class1_prev = prev[1]
+                Px_train = class1_prev * Pxy1_density + (1 - class1_prev) * Pxy0_density
+                return F.HellingerDistance(Px_train, Px_test)
+
+            prev_estimations.append(F.linear_search(loss, n_classes=2)[1])
 
         class1_prev = np.median(prev_estimations)
         return F.as_binary_prevalence(class1_prev)
@@ -1168,6 +1257,10 @@ class DMy(AggregativeSoftQuantifier):
 
     :param cdf: whether to use CDF instead of PDF (default False)
 
+    :param search: string indicating the search strategy used to estimate the prevalence values.
+        Valid options are `optim_minimize` (default, works for binary and multiclass problems),
+        `linear_search` (binary only), and `ternary_search` (binary only)
+
     :param n_jobs: number of parallel workers (default None)
     """
 
@@ -1218,7 +1311,9 @@ class DMy(AggregativeSoftQuantifier):
         :param labels: array-like with the true labels associated to each posterior
         """
         posteriors, true_labels = classif_predictions, labels
-        n_classes = len(self.classifier.classes_)
+        classes = self.classifier.classes_
+        n_classes = len(classes)
+        true_labels = _labels_to_indices(true_labels, classes)
 
         self.validation_distribution = qp.util.parallel(
             func=self._get_distributions,
diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py
index 6f204e4..706d9a9 100644
--- a/quapy/method/non_aggregative.py
+++ b/quapy/method/non_aggregative.py
@@ -9,6 +9,7 @@ from sklearn.preprocessing import normalize
 from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC
 from quapy.functional import get_divergence
 from quapy.method.base import BaseQuantifier, BinaryQuantifier
+from quapy.method._helper import _labels_to_indices
 import quapy.functional as F
 from scipy.optimize import lsq_linear
 from scipy import sparse
@@ -58,6 +59,9 @@ class DMx(BaseQuantifier):
         or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger
         Distance)
     :param cdf: whether to use CDF instead of PDF (default False)
+    :param search: string indicating the search strategy used to estimate the prevalence values.
+        Valid options are `optim_minimize` (default, works for binary and multiclass problems),
+        `linear_search` (binary only), and `ternary_search` (binary only)
     :param n_jobs: number of parallel workers (default None)
     """
 
@@ -122,7 +126,9 @@ class DMx(BaseQuantifier):
         """
         self.nfeats = X.shape[1]
         self.feat_ranges = _get_features_range(X)
-        n_classes = len(np.unique(y))
+        classes = np.unique(y)
+        y = _labels_to_indices(y, classes)
+        n_classes = len(classes)
 
         self.validation_distribution = np.asarray(
             [self.__get_distributions(X[y==cat]) for cat in range(n_classes)]
diff --git a/quapy/tests/test_methods.py b/quapy/tests/test_methods.py
index 9c07b33..91d23d5 100644
--- a/quapy/tests/test_methods.py
+++ b/quapy/tests/test_methods.py
@@ -2,10 +2,13 @@ import itertools
 import inspect
 import unittest
 
+import numpy as np
+
 from sklearn.linear_model import LogisticRegression
 
 from quapy.method import AGGREGATIVE_METHODS, BINARY_METHODS, NON_AGGREGATIVE_METHODS
-from quapy.method.aggregative import ACC
+from quapy.method.non_aggregative import DMx
+from quapy.method.aggregative import ACC, DMy, KDEyCS, RLLS
 from quapy.method.meta import Ensemble
 from quapy.functional import check_prevalence_vector
 from quapy.tests._synthetic import make_dataset
@@ -16,6 +19,7 @@ OPTIONAL_AGGREGATIVE_METHODS = {
     'BayesianKDEy',
     'BayesianMAPLS',
     'PQ',
+    'RLLS',
 }
 
 
@@ -122,6 +126,54 @@ class TestMethods(unittest.TestCase):
             from quapy.method.composable import __old_version_message
             print(__old_version_message)
 
+    def test_rlls(self):
+        try:
+            import cvxpy  # noqa: F401
+        except ImportError:
+            return
+
+        dataset = TestMethods.tiny_dataset_multiclass
+        q = RLLS(LogisticRegression(max_iter=2000), val_split=3)
+        q.fit(*dataset.training.Xy)
+        estim_prevalences = q.predict(dataset.test.X)
+        self.assertTrue(check_prevalence_vector(estim_prevalences))
+
+
+    def test_dmy_noncanonical_labels(self):
+        dataset = TestMethods.tiny_dataset_multiclass
+        label_names = np.asarray(['class-a', 'class-c', 'class-z'])
+        y_train = label_names[dataset.training.y]
+        y_test = label_names[dataset.test.y]
+
+        q = DMy(LogisticRegression(max_iter=2000), val_split=3)
+        q.fit(dataset.training.X, y_train)
+        estim_prevalences = q.predict(dataset.test.X)
+        self.assertTrue(check_prevalence_vector(estim_prevalences))
+        self.assertEqual(len(estim_prevalences), len(np.unique(y_test)))
+
+
+    def test_dmx_noncanonical_labels(self):
+        dataset = TestMethods.tiny_dataset_multiclass
+        label_names = np.asarray(['class-a', 'class-c', 'class-z'])
+        y_train = label_names[dataset.training.y]
+
+        q = DMx()
+        q.fit(dataset.training.X, y_train)
+        estim_prevalences = q.predict(dataset.test.X)
+        self.assertTrue(check_prevalence_vector(estim_prevalences))
+        self.assertEqual(len(estim_prevalences), len(np.unique(y_train)))
+
+    def test_kdeycs_noncanonical_labels(self):
+        dataset = TestMethods.tiny_dataset_multiclass
+        label_names = np.asarray(['class-a', 'class-c', 'class-z'])
+        y_train = label_names[dataset.training.y]
+
+        q = KDEyCS(LogisticRegression(max_iter=2000), val_split=3)
+        q.fit(dataset.training.X, y_train)
+        estim_prevalences = q.predict(dataset.test.X)
+        self.assertTrue(check_prevalence_vector(estim_prevalences))
+        self.assertEqual(len(estim_prevalences), len(np.unique(y_train)))
+
 
 if __name__ == '__main__':
     unittest.main()

From 4916919833c09fa41449858570a1c4340d699210 Mon Sep 17 00:00:00 2001
From: Alejandro Moreo 
Date: Thu, 2 Jul 2026 18:21:35 +0200
Subject: [PATCH 27/43] added dataset for images

---
 CHANGE_LOG.txt                                |   3 +
 docs/build/html/_modules/index.html           |  47 +-
 .../_modules/quapy/classification/neural.html | 608 +++++++++++----
 .../html/_modules/quapy/data/datasets.html    | 615 +++++++++++++---
 .../build/html/_modules/quapy/functional.html | 505 ++++++++++---
 .../html/_modules/quapy/method/_kdey.html     | 487 +++++++++---
 .../html/_modules/quapy/method/_neural.html   | 626 ++++++++++++----
 .../_modules/quapy/method/aggregative.html    | 542 +++++++++++---
 .../quapy/method/non_aggregative.html         | 502 ++++++++++---
 docs/build/html/_modules/quapy/plot.html      | 697 +++++++++++++++---
 docs/build/html/_modules/quapy/util.html      | 514 ++++++++++---
 docs/source/conf.py                           |  20 +-
 docs/source/index.md                          | 135 ++--
 docs/source/manuals/datasets.md               | 107 ++-
 docs/source/manuals/plotting.md               |  34 +
 quapy/data/datasets.py                        | 100 +++
 quapy/method/_kdey.py                         |   6 +-
 quapy/method/aggregative.py                   |  40 +-
 quapy/method/non_aggregative.py               |  22 +-
 quapy/plot.py                                 | 220 +++++-
 quapy/tests/test_methods.py                   |  16 +-
 quapy/util.py                                 |   2 +-
 22 files changed, 4861 insertions(+), 987 deletions(-)

diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt
index 06fb255..e1c61d4 100644
--- a/CHANGE_LOG.txt
+++ b/CHANGE_LOG.txt
@@ -6,12 +6,15 @@ Change Log 0.2.1
 - Added temperature calibration utilities for Bayesian confidence-aware methods.
 - Added compositional CLR and ILR transformations.
 - Extended KDEy with Aitchison/ILR kernels, shrinkage, and improved numerical stability.
+- Added image-embedding-based datasets including CIFAR10, CIFAR100, CIFAR100coarse, VSHN, FashionMNIST, MNIST
 - Added TemperatureScalingFromLogits for calibrating pretrained logits.
 - Added DirichletProtocol for prevalence sampling from Dirichlet priors.
 - Added ReadMe method by Daniel Hopkins and Gary King.
 - Internal index in LabelledCollection is now "lazy", and is only constructed if required.
 - Improved unit testing and separated integration tests.
 - Added RLLS (Regularized Learning for Domain Adaptation under Label Shifts) method.
+- Added visualization tools for 3-class problems in the simplex, see also the new example no.19
+
 
 Change Log 0.2.0
 -----------------
diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html
index ab28d32..10f0751 100644
--- a/docs/build/html/_modules/index.html
+++ b/docs/build/html/_modules/index.html
@@ -30,6 +30,8 @@
 
 
     
+    
+    
   
   
   
@@ -40,6 +42,7 @@
     
     
     
+    
     
     
     
@@ -113,8 +116,14 @@
   
   
   
+    
+    
+      
+    
+    
+    QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - Home
+    QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - Home
   
-    
QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation

   
 

     
@@ -130,7 +139,7 @@
     
 
  • 
   
-    Quickstart
+    Home
   
 
    • 
 
@@ -182,6 +191,21 @@
   
 
    
       
+        
      
+        
    • 
+          
+          
+          
+          
+          
+          
+          
+          
+          
+        
      • 
+
    
+      
     
    
     
   
    
@@ -228,7 +252,7 @@
     
 
  • 
   
-    Quickstart
+    Home
   
 
    • 
 
@@ -271,6 +295,21 @@
   
 
    
         
+          
      
+        
    • 
+          
+          
+          
+          
+          
+          
+          
+          
+          
+        
      • 
+
    
+
    @@ -330,6 +369,7 @@

    All modules for which code is available

    • quapy.classification.calibration
    • quapy.classification.methods
      • +
    • quapy.classification.neural
    • quapy.classification.svmperf
    • quapy.data.base
    • quapy.data.datasets
      • @@ -339,6 +379,7 @@
    • quapy.evaluation
    • quapy.functional
    • quapy.method._kdey
      • +
    • quapy.method._neural
    • quapy.method._threshold_optim
    • quapy.method.aggregative
    • quapy.method.base
      • diff --git a/docs/build/html/_modules/quapy/classification/neural.html b/docs/build/html/_modules/quapy/classification/neural.html index fd5d9b1..e5788fa 100644 --- a/docs/build/html/_modules/quapy/classification/neural.html +++ b/docs/build/html/_modules/quapy/classification/neural.html @@ -1,95 +1,395 @@ + - - - - - quapy.classification.neural — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.classification.neural — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
      - + + + + +
      +
      + + + + + + +
      + + + +
      + + +
      + + + +
      +
      + + +
      + + + +
      + +
      + +
      + +
        +
        • +
        • +
        • +
      +
      + +
        +
      • + + + + + + + + + +
        • +
      + +
      + +
      + + +
      +
      +
      +
      +
      + + +
      + +
      + + +
      +
      + +
      +
      + +
      + +
      + + +
      + +
      + + +
      +
      + + + + + +
      +

      Source code for quapy.classification.neural

      -import os
-from abc import ABCMeta, abstractmethod
-from pathlib import Path
+import os
+from abc import ABCMeta, abstractmethod
+from pathlib import Path
 
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from sklearn.metrics import accuracy_score, f1_score
-from torch.nn.utils.rnn import pad_sequence
-from tqdm import tqdm
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from sklearn.metrics import accuracy_score, f1_score
+from torch.nn.utils.rnn import pad_sequence
+from tqdm import tqdm
 
-import quapy as qp
-from quapy.data import LabelledCollection
-from quapy.util import EarlyStop
+import quapy as qp
+from quapy.data import LabelledCollection
+from quapy.util import EarlyStop
 
 
 
      
 [docs]
-class NeuralClassifierTrainer:
+class NeuralClassifierTrainer:
     """
     Trains a neural network for text classification.
 
@@ -107,7 +407,7 @@
         according to the evaluation in the held-out validation split (default '../checkpoint/classifier_net.dat')
     """
 
-    def __init__(self,
+    def __init__(self,
                  net: 'TextClassifierNet',
                  lr=1e-3,
                  weight_decay=0,
@@ -142,7 +442,7 @@
 
 
      
 [docs]
-    def reset_net_params(self, vocab_size, n_classes):
+    def reset_net_params(self, vocab_size, n_classes):
         """Reinitialize the network parameters
 
         :param vocab_size: the size of the vocabulary
@@ -155,7 +455,7 @@
 
 
      
 [docs]
-    def get_params(self):
+    def get_params(self):
         """Get hyper-parameters for this estimator
 
         :return: a dictionary with parameter names mapped to their values
@@ -165,7 +465,7 @@
 
 
      
 [docs]
-    def set_params(self, **params):
+    def set_params(self, **params):
         """Set the parameters of this trainer and the learner it is training.
         In this current version, parameter names for the trainer and learner should
         be disjoint.
@@ -191,14 +491,14 @@
 
 
     @property
-    def device(self):
+    def device(self):
         """ Gets the device in which the network is allocated
 
         :return: device
         """
         return next(self.net.parameters()).device
 
-    def _train_epoch(self, data, status, pbar, epoch):
+    def _train_epoch(self, data, status, pbar, epoch):
         self.net.train()
         criterion = torch.nn.CrossEntropyLoss()
         losses, predictions, true_labels = [], [], []
@@ -218,7 +518,7 @@
             status["f1"] = f1_score(true_labels, predictions, average='macro')
             self.__update_progress_bar(pbar, epoch)
 
-    def _test_epoch(self, data, status, pbar, epoch):
+    def _test_epoch(self, data, status, pbar, epoch):
         self.net.eval()
         criterion = torch.nn.CrossEntropyLoss()
         losses, predictions, true_labels = [], [], []
@@ -236,7 +536,7 @@
             status["f1"] = f1_score(true_labels, predictions, average='macro')
             self.__update_progress_bar(pbar, epoch)
 
-    def __update_progress_bar(self, pbar, epoch):
+    def __update_progress_bar(self, pbar, epoch):
         pbar.set_description(f'[{self.net.__class__.__name__}] training epoch={epoch} '
                              f'tr-loss={self.status["tr"]["loss"]:.5f} '
                              f'tr-acc={100 * self.status["tr"]["acc"]:.2f}% '
@@ -248,7 +548,7 @@
 
 
      
 [docs]
-    def fit(self, instances, labels, val_split=0.3):
+    def fit(self, instances, labels, val_split=0.3):
         """
         Fits the model according to the given training data.
 
@@ -297,7 +597,7 @@
 
 
      
 [docs]
-    def predict(self, instances):
+    def predict(self, instances):
         """
         Predicts labels for the instances
 
@@ -310,7 +610,7 @@
 
 
      
 [docs]
-    def predict_proba(self, instances):
+    def predict_proba(self, instances):
         """
         Predicts posterior probabilities for the instances
 
@@ -329,7 +629,7 @@
 
 
      
 [docs]
-    def transform(self, instances):
+    def transform(self, instances):
         """
         Returns the embeddings of the instances
 
@@ -351,7 +651,7 @@
 
 
      
 [docs]
-class TorchDataset(torch.utils.data.Dataset):
+class TorchDataset(torch.utils.data.Dataset):
     """
     Transforms labelled instances into a Torch's :class:`torch.utils.data.DataLoader` object
 
@@ -359,19 +659,19 @@
     :param labels: array-like of shape `(n_samples, n_classes)` with the class labels
     """
 
-    def __init__(self, instances, labels=None):
+    def __init__(self, instances, labels=None):
         self.instances = instances
         self.labels = labels
 
-    def __len__(self):
+    def __len__(self):
         return len(self.instances)
 
-    def __getitem__(self, index):
+    def __getitem__(self, index):
         return {'doc': self.instances[index], 'label': self.labels[index] if self.labels is not None else None}
 
 
      
 [docs]
-    def asDataloader(self, batch_size, shuffle, pad_length, device):
+    def asDataloader(self, batch_size, shuffle, pad_length, device):
         """
         Converts the labelled collection into a Torch DataLoader with dynamic padding for
         the batch
@@ -384,7 +684,7 @@
         :param device: whether to allocate tensors in cpu or in cuda
         :return: a :class:`torch.utils.data.DataLoader` object
         """
-        def collate(batch):
+        def collate(batch):
             data = [torch.LongTensor(item['doc'][:pad_length]) for item in batch]
             data = pad_sequence(data, batch_first=True, padding_value=qp.environ['PAD_INDEX']).to(device)
             targets = [item['label'] for item in batch]
@@ -402,7 +702,7 @@
 
 
      
 [docs]
-class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta):
+class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta):
     """
     Abstract Text classifier (`torch.nn.Module`)
     """
@@ -410,7 +710,7 @@
 
      
 [docs]
     @abstractmethod
-    def document_embedding(self, x):
+    def document_embedding(self, x):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -425,7 +725,7 @@
 
 
      
 [docs]
-    def forward(self, x):
+    def forward(self, x):
         """Performs the forward pass.
 
         :param x: a batch of instances, typically generated by a torch's `DataLoader`
@@ -439,7 +739,7 @@
 
 
      
 [docs]
-    def dimensions(self):
+    def dimensions(self):
         """Gets the number of dimensions of the embedding space
 
         :return: integer
@@ -449,7 +749,7 @@
 
 
      
 [docs]
-    def predict_proba(self, x):
+    def predict_proba(self, x):
         """
         Predicts posterior probabilities for the instances in `x`
 
@@ -464,7 +764,7 @@
 
 
      
 [docs]
-    def xavier_uniform(self):
+    def xavier_uniform(self):
         """
         Performs Xavier initialization of the network parameters
         """
@@ -476,7 +776,7 @@
 
      
 [docs]
     @abstractmethod
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -486,7 +786,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -498,7 +798,7 @@
 
 
      
 [docs]
-class LSTMnet(TextClassifierNet):
+class LSTMnet(TextClassifierNet):
     """
     An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on
     Long Short Term Memory networks.
@@ -512,7 +812,7 @@
     :param drop_p: drop probability for dropout (default 0.5)
     """
 
-    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1,
+    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1,
                  drop_p=0.5):
 
         super().__init__()
@@ -534,7 +834,7 @@
         self.doc_embedder = torch.nn.Linear(hidden_size, self.dim)
         self.output = torch.nn.Linear(self.dim, n_classes)
 
-    def __init_hidden(self, set_size):
+    def __init_hidden(self, set_size):
         opt = self.hyperparams
         var_hidden = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size'])
         var_cell = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size'])
@@ -544,7 +844,7 @@
 
 
      
 [docs]
-    def document_embedding(self, x):
+    def document_embedding(self, x):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -563,7 +863,7 @@
 
 
      
 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -573,7 +873,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -585,7 +885,7 @@
 
 
      
 [docs]
-class CNNnet(TextClassifierNet):
+class CNNnet(TextClassifierNet):
     """
     An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on
     Convolutional Neural Networks.
@@ -602,7 +902,7 @@
     :param drop_p: drop probability for dropout (default 0.5)
     """
 
-    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100,
+    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100,
                  kernel_heights=[3, 5, 7], stride=1, padding=0, drop_p=0.5):
         super(CNNnet, self).__init__()
 
@@ -627,7 +927,7 @@
         self.doc_embedder = torch.nn.Linear(len(kernel_heights) * hidden_size, self.dim)
         self.output = nn.Linear(self.dim, n_classes)
 
-    def __conv_block(self, input, conv_layer):
+    def __conv_block(self, input, conv_layer):
         conv_out = conv_layer(input)  # conv_out.size() = (batch_size, out_channels, dim, 1)
         activation = F.relu(conv_out.squeeze(3))  # activation.size() = (batch_size, out_channels, dim1)
         max_out = F.max_pool1d(activation, activation.size()[2]).squeeze(2)  # maxpool_out.size() = (batch_size, out_channels)
@@ -635,7 +935,7 @@
 
 
      
 [docs]
-    def document_embedding(self, input):
+    def document_embedding(self, input):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -660,7 +960,7 @@
 
 
      
 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -670,7 +970,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -685,31 +985,75 @@
      -
      + + + + + + +
      + +
      +
      +
      + +
      + + + +
      -
      - -
      - -
      -

      © Copyright 2024, Alejandro Moreo.

      +
      + +
      + + +
      + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
      +
      + +
      + +
      -
      -
      -
    - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/data/datasets.html b/docs/build/html/_modules/quapy/data/datasets.html index cbddc09..20d9588 100644 --- a/docs/build/html/_modules/quapy/data/datasets.html +++ b/docs/build/html/_modules/quapy/data/datasets.html @@ -1,78 +1,374 @@ - - - - - - quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
      +
    • + + + + + + + + + +
      • +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.data.datasets

     import os
 from contextlib import contextmanager
@@ -195,6 +491,9 @@
     'T4': 250,
 }
 
+IMAGE_DATASETS=['cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist']
+IMAGE_EMBEDDINGS=['features', 'logits', 'predictions']
+
 
 
    
 [docs]
@@ -1165,33 +1464,177 @@
     else:
         return train_gen, test_gen
    
 
+
+
+def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]:
+    """
+    Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo <https://zenodo.org/records/21131944>`_.
+    
+    Embeddings were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. 
+
+    :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'
+    :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation values), 'predictions' (posterior probabilities)
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quay_data/ directory)
+    :return: a tuple (train, val, test) where each entry is an instance of :class:`quapy.data.base.LabelledCollection`
+    """
+    assert dataset_name in IMAGE_DATASETS, \
+        f'Name {dataset_name} does not match any known dataset. Valid ones are {IMAGE_DATASETS}'
+    assert embedding in IMAGE_EMBEDDINGS, \
+        f'Name {embedding} does not match any known type of embedding. Valid ones are {IMAGE_EMBEDDINGS}'
+    if data_home is None:
+        data_home = get_quapy_home()
+    
+    dataset_network = {
+        'cifar10': 'resnet18',
+        'cifar100': 'resnet18',
+        'cifar100coarse': 'resnet18',
+        'svhn': 'resnet18',
+        'fashionmnist': 'basiccnn',
+        'mnist': 'basiccnn',
+    }
+
+    trained_network = dataset_network[dataset_name]
+
+    def download_embedding_npz(dataset_name, trained_network, embedding):
+        target_file = f'{dataset_name}_{trained_network}_{embedding}.npz'
+        URL = f'https://zenodo.org/records/21131944/files/{target_file}'
+        os.makedirs(join(data_home, 'image'), exist_ok=True)
+        file_path = join(data_home, 'image', target_file)
+        download_file_if_not_exists(URL, file_path)
+        npz_file = np.load(file_path)
+        return npz_file
+
+    embedding_dict = download_embedding_npz(dataset_name, trained_network, embedding=embedding)
+    labels_dict = download_embedding_npz(dataset_name, trained_network, embedding='targets')
+
+    train = LabelledCollection(embedding_dict['train'], labels_dict['train'])
+    val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes)
+    test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes)
+
+    print(f'{len(train)} | {len(val)} | {len(test)} | {train.X.shape[1]} | {train.n_classes} | {train.n_classes}')
+
+    return train, val, test
+
+
+
    
+[docs]
+def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset:
+    """
+    Loads an image dataset with pre-generated embeddings. Available datasets include:
+
+    - 'cifar10', 'cifar100', 'cifar100coarse': see `Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, Toronto, Ontario, 2009. <https://cave.cs.toronto.edu/kriz/learning-features-2009-TR.pdf>`_
+    - 'mnist': `Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998. <http://yann.lecun.com/exdb/mnist/>`_
+    - 'fashionmnist': `Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017. <https://arxiv.org/abs/1708.07747>`_
+    - 'svhn': `Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y Ng, et al. Reading digits in natural images with unsupervised feature learning. In NIPS workshop on deep learning and unsupervised feature learning, volume 2011, page 4. Granada, 2011. <https://static.googleusercontent.com/media/research.google.com/es//pubs/archive/37648.pdf>`_
+    
+    The image dataset are stored in `Zenodo <https://zenodo.org/records/21131944>`_ and were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. 
+
+    These embeddings were generated using a resnet18 or a simple cnn. In all cases, the network was trained using ~60% of the data, validated on ~25% of the data, and the remaining ~15% was used for test. Splits were created with stratification.
+    Once the network is trained, it was used with frozen weights to generate embeddings for the training, validation, and test, in different formats (see below).
+    It would therefore be convenient to use only heldout data (validation and test) for training and testing quantifiers (this is the default behavior), although the training+validation data can be accessed with `heldout_only=False`.
+
+    :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'
+    :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation outputs), 'predictions' (post-softmax outputs, or predicted posterior probabilities)
+    :param heldout_only: whether to discard the part of the training data used to train the neural model that generated the embeddings (default: True); set to False
+        to obtain, as the training data, the original training+validation splits.
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quay_data/ directory)
+    :return: an instance of :class:`quapy.data.base.Dataset`
+    """
+    if data_home is None:
+        data_home = get_quapy_home()
+
+    network_train, val, test = _fetch_image_embedding_splits(dataset_name, embedding, data_home)        
+
+    if heldout_only:
+        train = val            
+    else:
+        train = network_train + val
+        
+    return Dataset(train, test, name=dataset_name)
    
+
+
+
+if __name__ == '__main__':
+    #train, val, test = _fetch_image_embedding_splits(dataset_name='mnist', embedding='logits')
+    #print(train)
+    #print(val)
+    #print(test)
+
+    dataset = fetch_image_embeddings(dataset_name='svhn', embedding='features', heldout_only=True)
+    print(dataset)
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/functional.html b/docs/build/html/_modules/quapy/functional.html index 482f621..b7743b8 100644 --- a/docs/build/html/_modules/quapy/functional.html +++ b/docs/build/html/_modules/quapy/functional.html @@ -1,78 +1,336 @@ - - - - - - quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.functional

     import warnings
 from abc import ABC, abstractmethod
@@ -568,7 +826,7 @@
     :param method: string indicating the search strategy. Possible values are::
         'optim_minimize': uses scipy.optim
         'linear_search': carries out a linear search for binary problems in the space [0, 0.01, 0.02, ..., 1]
-        'ternary_search': implements the ternary search (not yet implemented)
+        'ternary_search': carries out a ternary search for binary problems in the interval [0,1]
     :return: np.ndarray, a prevalence vector
     """
     if method == 'optim_minimize':
@@ -576,7 +834,7 @@
     elif method == 'linear_search':
         return linear_search(loss, n_classes)
     elif method == 'ternary_search':
-        ternary_search(loss, n_classes)
+        return ternary_search(loss, n_classes)
     else:
         raise NotImplementedError()
    @@ -640,7 +898,32 @@ + """ + Performs a ternary search for the best prevalence value in binary problems. + This search assumes the loss is unimodal over the interval [0,1]. + + :param loss: (callable) the function to minimize + :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector + :return: (ndarray) the best prevalence vector found + """ + assert n_classes == 2, 'ternary search is only available for binary problems' + + left, right = 0., 1. + tol = 1e-5 + while abs(right - left) >= tol: + left_third = left + (right - left) / 3 + right_third = right - (right - left) / 3 + + left_loss = loss(np.asarray([1 - left_third, left_third])) + right_loss = loss(np.asarray([1 - right_third, right_third])) + + if left_loss < right_loss: + right = right_third + else: + left = left_third + + prev = (left + right) / 2 + return np.asarray([1 - prev, prev])
    @@ -947,31 +1230,75 @@
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    - - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + +
    + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/_kdey.html b/docs/build/html/_modules/quapy/method/_kdey.html index 9d82912..6664ea9 100644 --- a/docs/build/html/_modules/quapy/method/_kdey.html +++ b/docs/build/html/_modules/quapy/method/_kdey.html @@ -1,78 +1,336 @@ - - - - - - quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.method._kdey

     import numpy as np
 from numbers import Real
@@ -80,6 +338,7 @@
 from sklearn.neighbors import KernelDensity
 
 import quapy as qp
+from quapy.method._helper import _labels_to_indices
 from quapy.method.aggregative import AggregativeSoftQuantifier
 import quapy.functional as F
 from scipy.special import logsumexp
@@ -229,7 +488,7 @@
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Kullback-Leibler divergence (KLD) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors show that minimizing the distribution mathing criterion for KLD is akin to performing
     maximum likelihood (ML).
 
@@ -333,7 +592,7 @@
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the squared Hellinger Disntace (HD) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors proposed a Monte Carlo approach for minimizing the divergence.
 
     The distribution matching optimization problem comes down to solving:
@@ -444,7 +703,7 @@
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Cauchy-Schwarz divergence (CS) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors proposed a Monte Carlo approach for minimizing the divergence.
 
     The distribution matching optimization problem comes down to solving:
@@ -502,13 +761,11 @@
 
         P, y = classif_predictions, labels
         n = len(self.classes_)
-
-        assert all(sorted(np.unique(y)) == np.arange(n)), \
-            'label name gaps not allowed in current implementation'
+        y = _labels_to_indices(y, self.classes_)
 
         # counts_inv keeps track of the relative weight of each datapoint within its class
         # (i.e., the weight in its KDE model)
-        counts_inv = 1 / (F.counts_from_labels(y, classes=self.classes_))
+        counts_inv = 1 / (F.counts_from_labels(y, classes=np.arange(n)))
 
         # tr_tr_sums corresponds to symbol \overline{B} in the paper
         tr_tr_sums = np.zeros(shape=(n,n), dtype=float)
@@ -566,31 +823,75 @@
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/_neural.html b/docs/build/html/_modules/quapy/method/_neural.html index 706a7cc..3e6c7c3 100644 --- a/docs/build/html/_modules/quapy/method/_neural.html +++ b/docs/build/html/_modules/quapy/method/_neural.html @@ -1,91 +1,392 @@ + - - - - - quapy.method._neural — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method._neural — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
      +
    • + + + + + + + + + +
      • +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.method._neural

    -import os
-from pathlib import Path
-import random
+import os
+from pathlib import Path
+import random
 
-import torch
-from torch.nn import MSELoss
-from torch.nn.functional import relu
+import torch
+from torch.nn import MSELoss
+from torch.nn.functional import relu
 
-from quapy.protocol import UPP
-from quapy.method.aggregative import *
-from quapy.util import EarlyStop
-from tqdm import tqdm
+from quapy.protocol import UPP
+from quapy.method.aggregative import *
+from quapy.util import EarlyStop
+from tqdm import tqdm
 
 
-
    [docs]class QuaNetTrainer(BaseQuantifier):
+
    
+[docs]
+class QuaNetTrainer(BaseQuantifier):
     """
     Implementation of `QuaNet <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_, a neural network for
     quantification. This implementation uses `PyTorch <https://pytorch.org/>`_ and can take advantage of GPU
@@ -100,7 +401,7 @@
     >>> # use samples of 100 elements
     >>> qp.environ['SAMPLE_SIZE'] = 100
     >>>
-    >>> # load the kindle dataset as text, and convert words to numerical indexes
+    >>> # load the Kindle dataset as text, and convert words to numerical indexes
     >>> dataset = qp.datasets.fetch_reviews('kindle', pickle=True)
     >>> qp.train.preprocessing.index(dataset, min_df=5, inplace=True)
     >>>
@@ -110,12 +411,14 @@
     >>>
     >>> # train QuaNet (QuaNet is an alias to QuaNetTrainer)
     >>> model = QuaNet(classifier, qp.environ['SAMPLE_SIZE'], device='cuda')
-    >>> model.fit(dataset.training)
-    >>> estim_prevalence = model.quantify(dataset.test.instances)
+    >>> model.fit(*dataset.training.Xy)
+    >>> estim_prevalence = model.predict(dataset.test.instances)
 
     :param classifier: an object implementing `fit` (i.e., that can be trained on labelled data),
         `predict_proba` (i.e., that can generate posterior probabilities of unlabelled examples) and
         `transform` (i.e., that can generate embedded representations of the unlabelled instances).
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param sample_size: integer, the sample size; default is None, meaning that the sample size should be
         taken from qp.environ["SAMPLE_SIZE"]
     :param n_epochs: integer, maximum number of training epochs
@@ -135,8 +438,9 @@
     :param device: string, indicate "cpu" or "cuda"
     """
 
-    def __init__(self,
+    def __init__(self,
                  classifier,
+                 fit_classifier=True,
                  sample_size=None,
                  n_epochs=100,
                  tr_iter_per_poch=500,
@@ -159,6 +463,7 @@
             f'the classifier {classifier.__class__.__name__} does not seem to be able to produce posterior probabilities ' \
                 f'since it does not implement the method "predict_proba"'
         self.classifier = classifier
+        self.fit_classifier = fit_classifier
         self.sample_size = qp._get_sample_size(sample_size)
         self.n_epochs = n_epochs
         self.tr_iter = tr_iter_per_poch
@@ -184,20 +489,23 @@
         self.__check_params_colision(self.quanet_params, self.classifier.get_params())
         self._classes_ = None
 
-
    [docs]    def fit(self, data: LabelledCollection, fit_classifier=True):
+
    
+[docs]
+    def fit(self, X, y):
         """
         Trains QuaNet.
 
-        :param data: the training data on which to train QuaNet. If `fit_classifier=True`, the data will be split in
+        :param X: the training instances on which to train QuaNet. If `fit_classifier=True`, the data will be split in
             40/40/20 for training the classifier, training QuaNet, and validating QuaNet, respectively. If
             `fit_classifier=False`, the data will be split in 66/34 for training QuaNet and validating it, respectively.
-        :param fit_classifier: if True, trains the classifier on a split containing 40% of the data
+        :param y: the labels of X
         :return: self
         """
+        data = LabelledCollection(X, y)
         self._classes_ = data.classes_
         os.makedirs(self.checkpointdir, exist_ok=True)
 
-        if fit_classifier:
+        if self.fit_classifier:
             classifier_data, unused_data = data.split_stratified(0.4)
             train_data, valid_data = unused_data.split_stratified(0.66)  # 0.66 split of 60% makes 40% and 20%
             self.classifier.fit(*classifier_data.Xy)
@@ -217,13 +525,13 @@
         train_data_embed = LabelledCollection(self.classifier.transform(train_data.instances), train_data.labels, self._classes_)
 
         self.quantifiers = {
-            'cc': CC(self.classifier).fit(None, fit_classifier=False),
-            'acc': ACC(self.classifier).fit(None, fit_classifier=False, val_split=valid_data),
-            'pcc': PCC(self.classifier).fit(None, fit_classifier=False),
-            'pacc': PACC(self.classifier).fit(None, fit_classifier=False, val_split=valid_data),
+            'cc': CC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'acc': ACC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'pcc': PCC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'pacc': PACC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
         }
         if classifier_data is not None:
-            self.quantifiers['emq'] = EMQ(self.classifier).fit(classifier_data, fit_classifier=False)
+            self.quantifiers['emq'] = EMQ(self.classifier, fit_classifier=False).fit(*valid_data.Xy)
 
         self.status = {
             'tr-loss': -1,
@@ -263,7 +571,8 @@
 
         return self
    
 
-    def _get_aggregative_estims(self, posteriors):
+
+    def _get_aggregative_estims(self, posteriors):
         label_predictions = np.argmax(posteriors, axis=-1)
         prevs_estim = []
         for quantifier in self.quantifiers.values():
@@ -274,9 +583,11 @@
 
         return prevs_estim
 
-
    [docs]    def quantify(self, instances):
-        posteriors = self.classifier.predict_proba(instances)
-        embeddings = self.classifier.transform(instances)
+
    
+[docs]
+    def predict(self, X):
+        posteriors = self.classifier.predict_proba(X)
+        embeddings = self.classifier.transform(X)
         quant_estims = self._get_aggregative_estims(posteriors)
         self.quanet.eval()
         with torch.no_grad():
@@ -286,7 +597,8 @@
             prevalence = prevalence.numpy().flatten()
         return prevalence
    
 
-    def _epoch(self, data: LabelledCollection, posteriors, iterations, epoch, early_stop, train):
+
+    def _epoch(self, data: LabelledCollection, posteriors, iterations, epoch, early_stop, train):
         mse_loss = MSELoss()
 
         self.quanet.train(mode=train)
@@ -336,12 +648,17 @@
                                      f'val-mseloss={self.status["va-loss"]:.5f} val-maeloss={self.status["va-mae"]:.5f} '
                                      f'patience={early_stop.patience}/{early_stop.PATIENCE_LIMIT}')
 
-
    [docs]    def get_params(self, deep=True):
+
    
+[docs]
+    def get_params(self, deep=True):
         classifier_params = self.classifier.get_params()
         classifier_params = {'classifier__'+k:v for k,v in classifier_params.items()}
         return {**classifier_params, **self.quanet_params}
    
 
-
    [docs]    def set_params(self, **parameters):
+
+
    
+[docs]
+    def set_params(self, **parameters):
         learner_params = {}
         for key, val in parameters.items():
             if key in self.quanet_params:
@@ -352,7 +669,8 @@
                 raise ValueError('unknown parameter ', key)
         self.classifier.set_params(**learner_params)
    
 
-    def __check_params_colision(self, quanet_params, learner_params):
+
+    def __check_params_colision(self, quanet_params, learner_params):
         quanet_keys = set(quanet_params.keys())
         learner_keys = set(learner_params.keys())
         intersection = quanet_keys.intersection(learner_keys)
@@ -360,25 +678,34 @@
             raise ValueError(f'the use of parameters {intersection} is ambiguous sine those can refer to '
                              f'the parameters of QuaNet or the learner {self.classifier.__class__.__name__}')
 
-
    [docs]    def clean_checkpoint(self):
+
    
+[docs]
+    def clean_checkpoint(self):
         """
         Removes the checkpoint
         """
         os.remove(self.checkpoint)
    
 
-
    [docs]    def clean_checkpoint_dir(self):
+
+
    
+[docs]
+    def clean_checkpoint_dir(self):
         """
         Removes anything contained in the checkpoint directory
         """
-        import shutil
+        import shutil
         shutil.rmtree(self.checkpointdir, ignore_errors=True)
    
 
+
     @property
-    def classes_(self):
+    def classes_(self):
         return self._classes_
    
 
 
-
    [docs]def mae_loss(output, target):
+
+
    
+[docs]
+def mae_loss(output, target):
     """
     Torch-like wrapper for the Mean Absolute Error
 
@@ -389,7 +716,10 @@
     return torch.mean(torch.abs(output - target))
    
 
 
-
    [docs]class QuaNetModule(torch.nn.Module):
+
+
    
+[docs]
+class QuaNetModule(torch.nn.Module):
     """
     Implements the `QuaNet <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_ forward pass.
     See :class:`QuaNetTrainer` for training QuaNet.
@@ -406,7 +736,7 @@
     :param order_by: integer, class for which the document embeddings are to be sorted
     """
 
-    def __init__(self,
+    def __init__(self,
                  doc_embedding_size,
                  n_classes,
                  stats_size,
@@ -441,10 +771,10 @@
         self.output = torch.nn.Linear(prev_size, n_classes)
 
     @property
-    def device(self):
+    def device(self):
         return torch.device('cuda') if next(self.parameters()).is_cuda else torch.device('cpu')
 
-    def _init_hidden(self):
+    def _init_hidden(self):
         directions = 2 if self.bidirectional else 1
         var_hidden = torch.zeros(self.nlayers * directions, 1, self.hidden_size)
         var_cell = torch.zeros(self.nlayers * directions, 1, self.hidden_size)
@@ -452,7 +782,9 @@
             var_hidden, var_cell = var_hidden.cuda(), var_cell.cuda()
         return var_hidden, var_cell
 
-
    [docs]    def forward(self, doc_embeddings, doc_posteriors, statistics):
+
    
+[docs]
+    def forward(self, doc_embeddings, doc_posteriors, statistics):
         device = self.device
         doc_embeddings = torch.as_tensor(doc_embeddings, dtype=torch.float, device=device)
         doc_posteriors = torch.as_tensor(doc_posteriors, dtype=torch.float, device=device)
@@ -482,7 +814,9 @@
         logits = self.output(abstracted).view(1, -1)
         prevalence = torch.softmax(logits, -1)
 
-        return prevalence
    
+        return prevalence
    
+
    
+
 
 
 
@@ -490,31 +824,75 @@
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/aggregative.html b/docs/build/html/_modules/quapy/method/aggregative.html index 307965d..a6b98d4 100644 --- a/docs/build/html/_modules/quapy/method/aggregative.html +++ b/docs/build/html/_modules/quapy/method/aggregative.html @@ -1,78 +1,336 @@ - - - - - - quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.method.aggregative

     from abc import ABC, abstractmethod
 from argparse import ArgumentError
@@ -100,6 +358,7 @@
     _rlls_predicted_marginal,
     _rlls_compute_3deltaC,
     _rlls_compute_weights,
+    _labels_to_indices,
 )
 
 
@@ -1226,6 +1485,10 @@
     class-conditional distributions of the posterior probabilities returned for the positive and negative validation
     examples, respectively. The parameters of the mixture thus represent the estimates of the class prevalence values.
 
+    This dedicated class is kept for backward compatibility as the historical
+    HDy implementation. The same historical preset is also available as
+    :meth:`DMy.HDy`.
+
     :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
         the one indicated in `qp.environ['DEFAULT_CLS']`
 
@@ -1277,9 +1540,6 @@
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
 
         prev_estimations = []
-        # for bins in np.linspace(10, 110, 11, dtype=int):  #[10, 20, 30, ..., 100, 110]
-        # Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)
-        # Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)
         for bins in self.bins:
             Pxy0_density = self.Pxy0_density[bins]
             Pxy1_density = self.Pxy1_density[bins]
@@ -1289,13 +1549,12 @@
             # the authors proposed to search for the prevalence yielding the best matching as a linear search
             # at small steps (modern implementations resort to an optimization procedure,
             # see class DistributionMatching)
-            prev_selected, min_dist = None, None
-            for prev in F.prevalence_linspace(grid_points=101, repeats=1, smooth_limits_epsilon=0.0):
-                Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density
-                hdy = F.HellingerDistance(Px_train, Px_test)
-                if prev_selected is None or hdy < min_dist:
-                    prev_selected, min_dist = prev, hdy
-            prev_estimations.append(prev_selected)
+            def loss(prev):
+                class1_prev = prev[1]
+                Px_train = class1_prev * Pxy1_density + (1 - class1_prev) * Pxy0_density
+                return F.HellingerDistance(Px_train, Px_test)
+
+            prev_estimations.append(F.linear_search(loss, n_classes=2)[1])
 
         class1_prev = np.median(prev_estimations)
         return F.as_binary_prevalence(class1_prev)
    @@ -1476,6 +1735,10 @@ :param cdf: whether to use CDF instead of PDF (default False) + :param search: string indicating the search strategy used to estimate the prevalence values. + Valid options are `optim_minimize` (default, works for binary and multiclass problems), + `linear_search` (binary only), and `ternary_search` (binary only) + :param n_jobs: number of parallel workers (default None) """ @@ -1488,13 +1751,37 @@ self.search = search self.n_jobs = n_jobs - # @classmethod - # def HDy(cls, classifier, val_split=5, n_jobs=None): - # from quapy.method.meta import MedianEstimator - # - # hdy = DMy(classifier=classifier, val_split=val_split, search='linear_search', divergence='HD') - # hdy = AggregativeMedianEstimator(hdy, param_grid={'nbins': np.linspace(10, 110, 11).astype(int)}, n_jobs=n_jobs) - # return hdy +
    +[docs] + @classmethod + def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None): + """ + Historical HDy preset expressed as a configuration of :class:`DMy`. + + This preset reproduces the original HDy setup by using Hellinger + distance, PDF matching, linear search, and a median sweep over + `nbins` in `[10, 20, ..., 110]`. + + :param classifier: a scikit-learn's BaseEstimator, or None + :param fit_classifier: whether to train the learner + :param val_split: validation specification for generating posteriors + :param n_jobs: number of parallel workers + :return: an instance of :class:`AggregativeMedianEstimator` configured + to reproduce the historical HDy preset + """ + base = cls( + classifier=classifier, + fit_classifier=fit_classifier, + val_split=val_split, + nbins=10, + divergence='HD', + cdf=False, + search='linear_search', + n_jobs=n_jobs, + ) + param_grid = {'nbins': np.linspace(10, 110, 11, dtype=int)} + return AggregativeMedianEstimator(base_quantifier=base, param_grid=param_grid, n_jobs=n_jobs)
    + def _get_distributions(self, posteriors): histograms = [] @@ -1528,7 +1815,9 @@ :param labels: array-like with the true labels associated to each posterior """ posteriors, true_labels = classif_predictions, labels - n_classes = len(self.classifier.classes_) + classes = self.classifier.classes_ + n_classes = len(classes) + true_labels = _labels_to_indices(true_labels, classes) self.validation_distribution = qp.util.parallel( func=self._get_distributions, @@ -1965,35 +2254,80 @@ SLD = EMQ DistributionMatchingY = DMy HellingerDistanceY = HDy +HistoricalHDy = DMy.HDy MedianSweep = MS MedianSweep2 = MS2
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    - - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/non_aggregative.html b/docs/build/html/_modules/quapy/method/non_aggregative.html index e02061b..11c56e1 100644 --- a/docs/build/html/_modules/quapy/method/non_aggregative.html +++ b/docs/build/html/_modules/quapy/method/non_aggregative.html @@ -1,78 +1,336 @@ - - - - - - quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.method.non_aggregative

     from itertools import product
 from tqdm import tqdm
@@ -85,6 +343,7 @@
 from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC
 from quapy.functional import get_divergence
 from quapy.method.base import BaseQuantifier, BinaryQuantifier
+from quapy.method._helper import _labels_to_indices
 import quapy.functional as F
 from scipy.optimize import lsq_linear
 from scipy import sparse
@@ -145,6 +404,9 @@
         or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger
         Distance)
     :param cdf: whether to use CDF instead of PDF (default False)
+    :param search: string indicating the search strategy used to estimate the prevalence values.
+        Valid options are `optim_minimize` (default, works for binary and multiclass problems),
+        `linear_search` (binary only), and `ternary_search` (binary only)
     :param n_jobs: number of parallel workers (default None)
     """
 
@@ -183,7 +445,6 @@
 
 
     def __get_distributions(self, X):
-
         histograms = []
         for feat_idx in range(self.nfeats):
             feature = X[:, feat_idx]
@@ -214,7 +475,9 @@
         """
         self.nfeats = X.shape[1]
         self.feat_ranges = _get_features_range(X)
-        n_classes = len(np.unique(y))
+        classes = np.unique(y)
+        y = _labels_to_indices(y, classes)
+        n_classes = len(classes)
 
         self.validation_distribution = np.asarray(
             [self.__get_distributions(X[y==cat]) for cat in range(n_classes)]
@@ -252,8 +515,6 @@
 
 
 
-
-
 
    
 [docs]
 class ReadMe(BaseQuantifier, WithConfidenceABC):
@@ -280,8 +541,8 @@
         ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which
         `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where
         `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes.
-        Of course, this approach is computationally prohibited for large `K`, so the computation is restricted to data
-        matrices with `K<=25` (although we recommend even smaller values of `K`). A much faster model is "naive", which
+        Of course, this approach is computationally prohibited for large `K`, so the authors advised against computing it
+        for matrices with `K>25` (although we recommend even smaller values of `K`). A much faster model is "naive", which
         considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this
         case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior).
     :param bootstrap_trials: int, number of bootstrap trials (default 300)
@@ -325,7 +586,6 @@
         self.rng = np.random.default_rng(self.random_state)
         self.classes_ = np.unique(y)
 
-
         Xsize = X.shape[0]
 
         # Bootstrap loop
@@ -405,14 +665,16 @@
             raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions '
                              f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}')
 
-        # we convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5)
+        # we first convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5);
+        # this will speed up subsequent comparisons a lot
         K = X.shape[1]
-        binary_powers = 1 << np.arange(K-1, -1, -1)  # (2^K, ..., 32, 16, 8, 4, 2, 1)
-        X_as_binary_numbers = X @ binary_powers
+        binary_powers = 1 << np.arange(K-1, -1, -1)     # (2^K, ..., 32, 16, 8, 4, 2, 1)
+        X_as_binary_numbers = X @ binary_powers         # e.g., [0 0 1 0 1] @ [16, 8, 4, 2, 1] = 5
 
         # count occurrences and compute probs
         counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float)
         probs = counts / counts.sum()
+
         return probs
 
     def _multinomial_distribution(self, X):
@@ -422,8 +684,6 @@
 
 
 
-
-
 def _get_features_range(X):
     feat_ranges = []
     ncols = X.shape[1]
@@ -437,34 +697,80 @@
 # aliases
 #---------------------------------------------------------------
 
+HDx = DMx.HDx
 DistributionMatchingX = DMx
+HellingerDistanceX = HDx
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/plot.html b/docs/build/html/_modules/quapy/plot.html index 4c0b05d..86b9d5a 100644 --- a/docs/build/html/_modules/quapy/plot.html +++ b/docs/build/html/_modules/quapy/plot.html @@ -1,88 +1,349 @@ - - - - - - quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.plot

     from collections import defaultdict
-import matplotlib.pyplot as plt
-from matplotlib.pyplot import get_cmap
-import numpy as np
-from matplotlib import cm
-from scipy.stats import ttest_ind_from_stats
-from matplotlib.ticker import ScalarFormatter
 import math
 
+from matplotlib import cm
+import matplotlib.colors as mcolors
+import matplotlib.pyplot as plt
+from matplotlib.colors import LinearSegmentedColormap, ListedColormap
+from matplotlib.pyplot import get_cmap
+from matplotlib.ticker import ScalarFormatter
+import numpy as np
+from scipy.stats import ttest_ind_from_stats
+
 import quapy as qp
 
 plt.rcParams['figure.figsize'] = [10, 6]
@@ -651,6 +912,126 @@
 
 
 
+
    
+[docs]
+def plot_simplex(
+        point_layers=None,
+        region_layers=None,
+        density_function=None,
+        density_color='#1f77b4',
+        density_alpha=1.0,
+        resolution=400,
+        class_names=None,
+        title=None,
+        show_legend=True,
+        legend_loc='lower center',
+        legend_bbox_to_anchor=(0.5, -0.08),
+        legend_ncol=2,
+        figsize=(6.8, 6.2),
+        class_name_fontsize=10,
+        title_fontsize=11,
+        legend_fontsize=9,
+        ax=None,
+        savepath=None):
+    """
+    Plots data on the ternary simplex for three-class quantification problems.
+
+    This utility is convenient for visualising prevalence vectors, posterior triplets,
+    confidence regions, or any other points that lie on the 2-dimensional probability
+    simplex. The plot can combine three optional layer types:
+
+    * `point_layers`: scatter layers for one or more prevalence clouds or reference points
+    * `region_layers`: shaded regions defined by callables on prevalence vectors
+    * `density_function`: a scalar function evaluated on the simplex and rendered as a heatmap
+
+    Each entry in `point_layers` is a dictionary with a mandatory `points` field
+    containing an array-like of shape `(n_points, 3)` or `(3,)`. Optional fields are
+    `label` for the legend and `style` for matplotlib scatter keyword arguments.
+
+    Each entry in `region_layers` is a dictionary with a mandatory `fn` field containing
+    a callable that receives prevalence vectors and returns region-membership scores.
+    Optional fields are `label`, `color`, and `alpha`.
+
+    :param point_layers: optional list of point-layer dictionaries
+    :param region_layers: optional list of region-layer dictionaries
+    :param density_function: optional callable receiving prevalence vectors and returning
+        scalar values
+    :param density_color: color used for the density heatmap
+    :param density_alpha: opacity for the density heatmap
+    :param resolution: number of grid steps per axis used for rendering regions and densities
+    :param class_names: optional list or tuple with the three class names
+    :param title: optional plot title
+    :param show_legend: whether to display the legend
+    :param legend_loc: location string passed to matplotlib for the legend
+    :param legend_bbox_to_anchor: optional legend anchor box
+    :param legend_ncol: number of legend columns
+    :param figsize: figure size used when `ax` is not provided
+    :param class_name_fontsize: fontsize used for simplex vertex labels
+    :param title_fontsize: fontsize used for the optional title
+    :param legend_fontsize: fontsize used for the legend
+    :param ax: optional matplotlib axes object; if not provided, a new figure is created
+    :param savepath: path where to save the plot; if not indicated, the plot is shown when
+        `ax` is not provided
+    :return: returns `(fig, ax)` matplotlib objects for eventual customisation
+    """
+    if class_names is None:
+        class_names = ('Y=1', 'Y=2', 'Y=3')
+    if len(class_names) != 3:
+        raise ValueError(f'expected exactly 3 class names, got {len(class_names)}')
+
+    own_figure = ax is None
+    if own_figure:
+        fig, ax = plt.subplots(figsize=figsize)
+    else:
+        fig = ax.figure
+
+    if density_function is not None:
+        _plot_simplex_density(ax, density_function, resolution, density_color, density_alpha)
+
+    if region_layers:
+        _plot_simplex_regions(ax, region_layers, resolution)
+
+    if point_layers:
+        _plot_simplex_points(ax, point_layers)
+
+    simplex_ymax = np.sqrt(3) / 2
+    triangle = np.array([
+        [0.0, 0.0],
+        [1.0, 0.0],
+        [0.5, simplex_ymax],
+        [0.0, 0.0],
+    ])
+    ax.plot(triangle[:, 0], triangle[:, 1], color='black')
+
+    ax.text(-0.05, -0.05, class_names[0], ha='right', va='top', fontsize=class_name_fontsize)
+    ax.text(1.05, -0.05, class_names[1], ha='left', va='top', fontsize=class_name_fontsize)
+    ax.text(0.5, simplex_ymax + 0.05, class_names[2], ha='center', va='bottom', fontsize=class_name_fontsize)
+
+    if title is not None:
+        ax.set_title(title, fontsize=title_fontsize)
+
+    ax.set_aspect('equal')
+    ax.set_xlim(-0.1, 1.1)
+    ax.set_ylim(-0.1, simplex_ymax + 0.1)
+    ax.axis('off')
+
+    if show_legend:
+        _, labels = ax.get_legend_handles_labels()
+        if labels:
+            ax.legend(loc=legend_loc, bbox_to_anchor=legend_bbox_to_anchor, ncol=legend_ncol, fontsize=legend_fontsize, frameon=False)
+
+    fig.tight_layout(pad=0.6)
+
+    if savepath is not None:
+        qp.util.create_parent_dir(savepath)
+        fig.savefig(savepath, bbox_inches='tight')
+    elif own_figure:
+        plt.show()
+
+    return fig, ax
    
+
+
+
 def _merge(method_names, true_prevs, estim_prevs):
     ndims = true_prevs[0].shape[1]
     data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))})
@@ -703,6 +1084,94 @@
     return data
 
 
+def _simplex_to_cartesian(prevalences):
+    prevalences = np.asarray(prevalences, dtype=float)
+    prevalences = np.atleast_2d(prevalences)
+    if prevalences.shape[1] != 3:
+        raise ValueError(f'plot_simplex expects prevalence vectors of shape (_, 3); found {prevalences.shape}')
+    x = prevalences[:, 1] + 0.5 * prevalences[:, 2]
+    y = prevalences[:, 2] * (np.sqrt(3) / 2)
+    return x, y
+
+
+def _barycentric_from_xy(x, y):
+    p3 = 2 * y / np.sqrt(3)
+    p2 = x - 0.5 * p3
+    p1 = 1 - p2 - p3
+    return np.stack([p1, p2, p3], axis=-1)
+
+
+def _simplex_mesh(resolution):
+    simplex_ymax = np.sqrt(3) / 2
+    xs = np.linspace(0, 1, resolution)
+    ys = np.linspace(0, simplex_ymax, resolution)
+    grid_x, grid_y = np.meshgrid(xs, ys)
+    pts_bary = _barycentric_from_xy(grid_x, grid_y)
+    mask = np.all(pts_bary >= 0, axis=-1)
+    return xs, ys, pts_bary, mask
+
+
+def _evaluate_simplex_function(function, points):
+    points = np.asarray(points, dtype=float)
+    try:
+        values = np.asarray(function(points), dtype=float)
+        if values.shape == (points.shape[0],):
+            return values
+        if values.shape == points.shape[:-1]:
+            return values.reshape(-1)
+    except Exception:
+        pass
+    return np.asarray([function(point) for point in points], dtype=float)
+
+
+def _region_colormap(color='blue', alpha=0.35):
+    return ListedColormap([
+        (1.0, 1.0, 1.0, 0.0),
+        (*mcolors.to_rgb(color), alpha),
+    ])
+
+
+def _plot_simplex_points(ax, point_layers):
+    for layer in point_layers:
+        points = np.asarray(layer['points'], dtype=float)
+        style = {'s': 25, 'alpha': 0.8}
+        style.update(layer.get('style', {}))
+        ax.scatter(*_simplex_to_cartesian(points), label=layer.get('label'), **style)
+
+
+def _plot_simplex_regions(ax, region_layers, resolution):
+    xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution)
+    valid_points = pts_bary[simplex_mask]
+
+    for layer in region_layers:
+        mask = np.zeros(simplex_mask.shape, dtype=float)
+        values = _evaluate_simplex_function(layer['fn'], valid_points)
+        mask[simplex_mask] = values
+        ax.pcolormesh(
+            xs,
+            ys,
+            mask,
+            shading='auto',
+            cmap=_region_colormap(layer.get('color', 'blue'), layer.get('alpha', 0.35)),
+        )
+        if layer.get('label') is not None:
+            ax.scatter([], [], color=layer.get('color', 'blue'), alpha=layer.get('alpha', 0.35), label=layer['label'])
+
+
+def _plot_simplex_density(ax, density_function, resolution, color, alpha):
+    xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution)
+    valid_points = pts_bary[simplex_mask]
+    density = np.full(simplex_mask.shape, np.nan, dtype=float)
+    values = _evaluate_simplex_function(density_function, valid_points)
+    min_v, max_v = np.min(values), np.max(values)
+    if max_v > min_v:
+        values = (values - min_v) / (max_v - min_v)
+    density[simplex_mask] = values
+
+    cmap = LinearSegmentedColormap.from_list('simplex_density', ['white', color])
+    ax.pcolormesh(xs, ys, density, shading='auto', cmap=cmap, alpha=alpha)
+
+
 
    
 [docs]
 def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None):
@@ -742,31 +1211,75 @@
     calibration_plot(classifier, *test.Xy)
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/util.html b/docs/build/html/_modules/quapy/util.html index 5f34290..9a26185 100644 --- a/docs/build/html/_modules/quapy/util.html +++ b/docs/build/html/_modules/quapy/util.html @@ -1,78 +1,374 @@ - - - - - - quapy.util — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - + + + + + + + + quapy.util — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - +
    + + + + + + + + +
    + + + Ctrl+K +
    +     - -
    - + + + + +
    +
    + + + + + + +
    + + + +
    + + +
    + + + +
    +
    + + +
    + + + +
    + +
    + +
    + +
      +
      • +
      • +
      • +
    +
    + +
      +
    • + + + + + + + + + +
      • +
    + +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + +
    + + +
    +
    + +
    +
    + +
    + +
    + + +
    + +
    + + +
    +
    + + + + + +
    +

    Source code for quapy.util

     import contextlib
 import itertools
@@ -245,7 +541,7 @@
 [docs]
 def download_file_if_not_exists(url, archive_filename):
     """
-    Dowloads a function (using :meth:`download_file`) if the file does not exist.
+    Downloads a file (using :meth:`download_file`) if the file does not exist.
 
     :param url: the url
     :param archive_filename: destination filename
@@ -479,31 +775,75 @@
    -
    + + + + + + +
    + +
    +
    +
    + +
    + + + +
    -
    - -
    - -
    -

    © Copyright 2024, Alejandro Moreo.

    +
    + +
    + + +
    + + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
    +
    + +
    + +
    -
    -
    - - - - +

    + + © Copyright 2024, Alejandro Moreo. +
    + +

    + + +
    - +

    + Created using Sphinx 9.0.4. +
    +

    +
    + + + + + +
    + +
    +

    + + Built with the PyData Sphinx Theme 0.19.0. +

    + +
    + + + + + \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 96270d5..a40a690 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,12 +44,15 @@ extensions = [ 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'myst_parser', + 'sphinx_design', ] autosectionlabel_prefix_document = True source_suffix = ['.rst', '.md'] +myst_enable_extensions = ['colon_fence'] + templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and @@ -65,7 +68,22 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] html_theme = 'pydata_sphinx_theme' # html_theme = 'furo' # need to be installed: pip install furo (not working...) -# html_static_path = ['_static'] +html_static_path = ['_static'] +html_css_files = ['custom.css'] +html_theme_options = { + 'logo': { + 'image_light': '_static/quapy_logo.png', + 'image_dark': '_static/quapy_logo_dark.png', + }, + 'icon_links': [ + { + 'name': 'GitHub', + 'url': 'https://github.com/HLT-ISTI/QuaPy', + 'icon': 'fa-brands fa-github', + 'type': 'fontawesome', + }, + ], +} # intersphinx configuration intersphinx_mapping = { diff --git a/docs/source/index.md b/docs/source/index.md index d52093e..a9e4b64 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,16 +1,68 @@ ```{toctree} :hidden: -self +Home +manuals +API ``` -# Quickstart +# QuaPy -QuaPy is an open source framework for quantification (a.k.a. supervised prevalence estimation, or learning to quantify) written in Python. +```{div} hero-copy +QuaPy is an open-source Python framework for quantification, also known as +supervised prevalence estimation or learning to quantify. It is designed with +research and experimental analysis in mind, and combines datasets, protocols, +evaluation measures, visualization tools, and a broad collection of +quantification methods in a single coherent workflow. +``` -QuaPy is based on the concept of "data sample", and provides implementations of the most important aspects of the quantification workflow, such as (baseline and advanced) quantification methods, quantification-oriented model selection mechanisms, evaluation measures, and evaluations protocols used for evaluating quantification methods. QuaPy also makes available commonly used datasets, and offers visualization tools for facilitating the analysis and interpretation of the experimental results. +`````{grid} 1 1 2 2 +:gutter: 3 +:class-container: landing-grid -QuaPy is hosted on GitHub at [https://github.com/HLT-ISTI/QuaPy](https://github.com/HLT-ISTI/QuaPy). +````{grid-item-card} Quickstart +:class-card: landing-card +Install QuaPy and run your first quantifier in a few lines of code. ++++ +```{button-link} #installation +:color: primary +Get Started +``` +```` + +````{grid-item-card} Manuals +:class-card: landing-card +Hands-on guides with methodological context, literature pointers, and reproducible workflows. ++++ +```{button-ref} manuals +:ref-type: doc +:color: primary +Open Manuals +``` +```` + +````{grid-item-card} API +:class-card: landing-card +Browse the full reference for `quapy`, including methods, datasets, utilities, and research-oriented extensions. ++++ +```{button-ref} quapy +:ref-type: doc +:color: primary +Browse API +``` +```` + +````{grid-item-card} GitHub +:class-card: landing-card +Explore the source code, open issues, and current development branch activity. ++++ +```{button-link} https://github.com/HLT-ISTI/QuaPy +:color: primary +Open GitHub +``` +```` + +````` ## Installation @@ -18,16 +70,37 @@ QuaPy is hosted on GitHub at [https://github.com/HLT-ISTI/QuaPy](https://github. pip install quapy ``` -## Usage +## Why QuaPy -The following script fetches a dataset of tweets, trains, applies, and evaluates a quantifier based on the *Adjusted Classify & Count* quantification method, using, as the evaluation measure, the *Mean Absolute Error* (MAE) between the predicted and the true class prevalence values of the test set: +QuaPy is built around the concept of a data sample and supports the main tasks +in the quantification workflow: training quantifiers, generating evaluation +samples, measuring quantification error, selecting models under distribution +shift, and visualizing experimental behaviour. The framework is especially +suited for research settings, where one often needs not only implementations, +but also methodological context, literature links, and reproducible evaluation +procedures. + +Some of the main features are: + +* Implementation of many popular quantification methods, including Classify & Count and its variants, + Expectation Maximization, HDy, QuaNet, quantification ensembles, and Bayesian extensions. +* Evaluation protocols for generating test samples under prior probability shift. +* A broad set of quantification-oriented evaluation metrics. +* Ready-to-use textual, numeric, and benchmark competition datasets. +* Method documentation that points back to the relevant literature and original papers. +* Native support for binary and single-label multiclass quantification. +* Visualization tools for analysing predictions, drift, confidence regions, and ternary prevalences. + +## First Example + +The following script fetches a binary dataset, trains an Adjusted Classify & Count quantifier, +and evaluates the resulting prevalence prediction with Mean Absolute Error. ```python import quapy as qp training, test = qp.datasets.fetch_UCIBinaryDataset("yeast").train_test -# create an "Adjusted Classify & Count" quantifier model = qp.method.aggregative.ACC() Xtr, ytr = training.Xy model.fit(Xtr, ytr) @@ -39,43 +112,14 @@ error = qp.error.mae(true_prevalence, estim_prevalence) print(f'Mean Absolute Error (MAE)={error:.3f}') ``` -Quantification is useful in scenarios characterized by prior probability shift. In other words, we would be little interested in estimating the class prevalence values of the test set if we could assume the IID assumption to hold, as this prevalence would be roughly equivalent to the class prevalence of the training set. For this reason, any quantification model should be tested across many samples, even ones characterized by class prevalence values different or very different from those found in the training set. QuaPy implements sampling procedures and evaluation protocols that automate this workflow. See the [](./manuals) for detailed examples. - -## Manuals - -The following manuals illustrate several aspects of QuaPy through examples: - -```{toctree} -:maxdepth: 3 - -manuals -``` - -```{toctree} -:hidden: - -API -``` - -## Features - -* Implementation of many popular quantification methods (Classify-&-Count and its variants, Expectation Maximization, -quantification methods based on structured output learning, HDy, QuaNet, quantification ensembles, among others). -* Versatile functionality for performing evaluation based on sampling generation protocols (e.g., APP, NPP, etc.). -* Implementation of most commonly used evaluation metrics (e.g., AE, RAE, NAE, NRAE, SE, KLD, NKLD, etc.). -* Datasets frequently used in quantification (textual and numeric), including: - * 32 UCI Machine Learning datasets. - * 11 Twitter quantification-by-sentiment datasets. - * 3 product reviews quantification-by-sentiment datasets. - * 4 tasks from LeQua 2022 competition and 4 tasks from LeQua 2024 competition - * IFCB for Plancton quantification -* Native support for binary and single-label multiclass quantification scenarios. -* Model selection functionality that minimizes quantification-oriented loss functions. -* Visualization tools for analysing the experimental results. +Quantification is especially useful when the class prevalence of the test data +may differ from that of the training data. QuaPy implements protocols that make +it easy to evaluate methods across many such shifts. See the [](./manuals) for +worked examples. ## Citing QuaPy -If you find QuaPy useful (and we hope you will), please consider citing the original paper in your research. +If you find QuaPy useful, please consider citing the original paper. ```bibtex @inproceedings{moreo2021quapy, @@ -89,7 +133,8 @@ If you find QuaPy useful (and we hope you will), please consider citing the orig ## Contributing -In case you want to contribute improvements to quapy, please generate pull request to the "devel" branch. +If you want to contribute improvements to QuaPy, please open a pull request +against the `devel` branch. ## Acknowledgments @@ -98,6 +143,6 @@ In case you want to contribute improvements to quapy, please generate pull reque :alt: SoBigData++ ``` -This work has been supported by the QuaDaSh project -_"Finanziato dall’Unione europea---Next Generation EU, +This work has been supported by the QuaDaSh project +_"Finanziato dall'Unione europea---Next Generation EU, Missione 4 Componente 2 CUP B53D23026250001"_. diff --git a/docs/source/manuals/datasets.md b/docs/source/manuals/datasets.md index b7d8827..dff406c 100644 --- a/docs/source/manuals/datasets.md +++ b/docs/source/manuals/datasets.md @@ -412,11 +412,112 @@ ECML-PKDD 2024, Vilnius, Lithuania. ``` +## Image Embedding Datasets + +QuaPy also provides a collection of image datasets in the form of pre-generated +embeddings, hosted in [Zenodo](https://zenodo.org/records/21131944). +These +embeddings were generated using [this extraction scripts](https://github.com/pglez82/visiondatasets_quapy). + +The current public interface is: + +```python +import quapy as qp + +data = qp.datasets.fetch_image_embeddings( + dataset_name='cifar10', + embedding='features', + heldout_only=True, +) +train, test = data.train_test +``` + +The available datasets are: + +```python +qp.datasets.IMAGE_DATASETS +# ['cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'] +``` + +The available embedding types are: + +```python +qp.datasets.IMAGE_EMBEDDINGS +# ['features', 'logits', 'predictions'] +``` + +where: + +* `features` are the penultimate-layer representations +* `logits` are the pre-activation outputs of the neural model +* `predictions` are the post-softmax posterior probabilities + +The datasets correspond to frozen neural representations extracted from models +trained on image classification tasks. QuaPy downloads them automatically on +first use and stores them locally for fast reuse. + +### Train/Test Semantics + +Each dataset is internally organised into three splits: `train`, `val`, and +`test`. The `train` split was used to train the neural model that produced the +embeddings, while `val` and `test` were not seen during neural training. + +For this reason, the default setting is: + +```python +data = qp.datasets.fetch_image_embeddings(..., heldout_only=True) +``` + +which returns: + +* `train = val` +* `test = test` + +This is often the most convenient choice for quantification experiments, since +it avoids training quantifiers on examples that were already used to train the +embedding model. + +If instead you want to use all the available non-test data, you can set: + +```python +data = qp.datasets.fetch_image_embeddings(..., heldout_only=False) +``` + +In this case, the returned training set is the union of the original neural +training split and the validation split. + +### Sources + +The image datasets currently available through `fetch_image_embeddings` are: + +* cifar10, cifar100, cifar100coarse: + [Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, 2009.](https://cave.cs.toronto.edu/kriz/learning-features-2009-TR.pdf) +* mnist: + [Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998.](http://yann.lecun.com/exdb/mnist/) +* fashionmnist: + [Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017.](https://arxiv.org/abs/1708.07747) +* svhn: + [Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y. Ng, et al. Reading digits in natural images with unsupervised feature learning. NIPS Workshop, 2011.](https://static.googleusercontent.com/media/research.google.com/es//pubs/archive/37648.pdf) + +Some statistics are shown in the following table: + +| Dataset | backbone | classes | neural network train size | validation size | test size | feature dim | logit dim | prediction dim | type | +|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|---| +| cifar100 | resnet18 | 100 | 35000 | 15000 | 10000 | 512 | 100 | 100 | dense | +| cifar10 | resnet18 | 10 | 35000 | 15000 | 10000 | 512 | 10 | 10 | dense | +| cifar100coarse | resnet18 | 20 | 35000 | 15000 | 10000 | 512 | 20 | 20 | dense | +| mnist | basiccnn | 10 |42000 | 18000 | 10000 | 128 | 10 | 10 | dense | +| fashionmnist | basiccnn | 10 | 42000 | 18000 | 10000 | 128 | 10 | 10 | dense | +| svhn | resnet18 | 10 | 51280 | 21977 | 26032 | 512 | 10 | 10 | dense | + + + + ## IFCB Plankton dataset -IFCB is a dataset of plankton species in water samples hosted in `Zenodo `_. -This dataset is based on the data available publicly at `WHOI-Plankton repo `_ -and in the scripts for the processing are available at `P. González's repo `_. +IFCB is a dataset of plankton species in water samples hosted in [Zenodo](https://zenodo.org/records/10036244). +This dataset is based on the data available publicly at [WHOI-Plankton repo](https://github.com/hsosik/WHOI-Plankton) +and the scripts for the processing are available at [P. González's repo](https://github.com/pglez82/IFCB_Zenodo). This dataset comes with precomputed features for testing quantification algorithms. diff --git a/docs/source/manuals/plotting.md b/docs/source/manuals/plotting.md index 67f9f16..cf685fe 100644 --- a/docs/source/manuals/plotting.md +++ b/docs/source/manuals/plotting.md @@ -251,3 +251,37 @@ In those cases, however, it is likely that the variances of each method get higher, to the detriment of the visualization. We recommend to set _show_std=False_ in those cases in order to hide the color bands. + +## Simplex Visualisation + +For three-class problems, prevalence vectors lie on the 2-dimensional probability simplex. +The function `qp.plot.plot_simplex` provides a lightweight ternary plot that can combine +scatter layers, shaded regions, and density overlays. + +A simplex plot is specified through optional layers. Point layers are dictionaries with +fields `points`, `label`, and `style`; region layers are dictionaries with fields `fn`, +`label`, `color`, and `alpha`. + +```python +import numpy as np +import quapy as qp + +true_prev = np.array([0.20, 0.35, 0.45]) +train_prev = np.array([0.50, 0.30, 0.20]) +posterior_cloud = np.random.default_rng(0).dirichlet(alpha=40 * true_prev, size=200) + +qp.plot.plot_simplex( + point_layers=[ + {'points': posterior_cloud, 'label': 'posterior cloud', 'style': {'s': 12, 'alpha': 0.25, 'color': 'steelblue'}}, + {'points': true_prev, 'label': 'true prevalence', 'style': {'s': 90, 'color': 'black'}}, + {'points': train_prev, 'label': 'training prevalence', 'style': {'s': 90, 'color': 'darkorange'}}, + ], + density_function=lambda p: np.exp(-40 * np.sum((p - true_prev) ** 2, axis=1)), + class_names=['class A', 'class B', 'class C'], + savepath='./plots/simplex.png', +) +``` + +See the dedicated +[example](https://github.com/HLT-ISTI/QuaPy/blob/master/examples/19.visualizing_simplex.py) +for a slightly richer illustration. diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index 8c235e3..63e9314 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -119,6 +119,9 @@ LEQUA2024_SAMPLE_SIZE = { 'T4': 250, } +IMAGE_DATASETS=['cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'] +IMAGE_EMBEDDINGS=['features', 'logits', 'predictions'] + def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset: """ @@ -1062,3 +1065,100 @@ def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=No return train, test_gen else: return train_gen, test_gen + + +def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]: + """ + Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo `_. + + Embeddings were extracted using `this script `_. + + :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist' + :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation values), 'predictions' (posterior probabilities) + :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default + ~/quay_data/ directory) + :return: a tuple (train, val, test) where each entry is an instance of :class:`quapy.data.base.LabelledCollection` + """ + assert dataset_name in IMAGE_DATASETS, \ + f'Name {dataset_name} does not match any known dataset. Valid ones are {IMAGE_DATASETS}' + assert embedding in IMAGE_EMBEDDINGS, \ + f'Name {embedding} does not match any known type of embedding. Valid ones are {IMAGE_EMBEDDINGS}' + if data_home is None: + data_home = get_quapy_home() + + dataset_network = { + 'cifar10': 'resnet18', + 'cifar100': 'resnet18', + 'cifar100coarse': 'resnet18', + 'svhn': 'resnet18', + 'fashionmnist': 'basiccnn', + 'mnist': 'basiccnn', + } + + trained_network = dataset_network[dataset_name] + + def download_embedding_npz(dataset_name, trained_network, embedding): + target_file = f'{dataset_name}_{trained_network}_{embedding}.npz' + URL = f'https://zenodo.org/records/21131944/files/{target_file}' + os.makedirs(join(data_home, 'image'), exist_ok=True) + file_path = join(data_home, 'image', target_file) + download_file_if_not_exists(URL, file_path) + npz_file = np.load(file_path) + return npz_file + + embedding_dict = download_embedding_npz(dataset_name, trained_network, embedding=embedding) + labels_dict = download_embedding_npz(dataset_name, trained_network, embedding='targets') + + train = LabelledCollection(embedding_dict['train'], labels_dict['train']) + val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes) + test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes) + + print(f'{len(train)} | {len(val)} | {len(test)} | {train.X.shape[1]} | {train.n_classes} | {train.n_classes}') + + return train, val, test + + +def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset: + """ + Loads an image dataset with pre-generated embeddings. Available datasets include: + + - 'cifar10', 'cifar100', 'cifar100coarse': see `Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, Toronto, Ontario, 2009. `_ + - 'mnist': `Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998. `_ + - 'fashionmnist': `Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017. `_ + - 'svhn': `Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y Ng, et al. Reading digits in natural images with unsupervised feature learning. In NIPS workshop on deep learning and unsupervised feature learning, volume 2011, page 4. Granada, 2011. `_ + + The image dataset are stored in `Zenodo `_ and were extracted using `this script `_. + + These embeddings were generated using a resnet18 or a simple cnn. In all cases, the network was trained using ~60% of the data, validated on ~25% of the data, and the remaining ~15% was used for test. Splits were created with stratification. + Once the network is trained, it was used with frozen weights to generate embeddings for the training, validation, and test, in different formats (see below). + It would therefore be convenient to use only heldout data (validation and test) for training and testing quantifiers (this is the default behavior), although the training+validation data can be accessed with `heldout_only=False`. + + :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist' + :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation outputs), 'predictions' (post-softmax outputs, or predicted posterior probabilities) + :param heldout_only: whether to discard the part of the training data used to train the neural model that generated the embeddings (default: True); set to False + to obtain, as the training data, the original training+validation splits. + :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default + ~/quay_data/ directory) + :return: an instance of :class:`quapy.data.base.Dataset` + """ + if data_home is None: + data_home = get_quapy_home() + + network_train, val, test = _fetch_image_embedding_splits(dataset_name, embedding, data_home) + + if heldout_only: + train = val + else: + train = network_train + val + + return Dataset(train, test, name=dataset_name) + + +if __name__ == '__main__': + #train, val, test = _fetch_image_embedding_splits(dataset_name='mnist', embedding='logits') + #print(train) + #print(val) + #print(test) + + dataset = fetch_image_embeddings(dataset_name='svhn', embedding='features', heldout_only=True) + print(dataset) \ No newline at end of file diff --git a/quapy/method/_kdey.py b/quapy/method/_kdey.py index 37a6501..8f12f24 100644 --- a/quapy/method/_kdey.py +++ b/quapy/method/_kdey.py @@ -125,7 +125,7 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): """ Kernel Density Estimation model for quantification (KDEy) relying on the Kullback-Leibler divergence (KLD) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors show that minimizing the distribution mathing criterion for KLD is akin to performing maximum likelihood (ML). @@ -220,7 +220,7 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): """ Kernel Density Estimation model for quantification (KDEy) relying on the squared Hellinger Disntace (HD) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors proposed a Monte Carlo approach for minimizing the divergence. The distribution matching optimization problem comes down to solving: @@ -322,7 +322,7 @@ class KDEyCS(AggregativeSoftQuantifier): """ Kernel Density Estimation model for quantification (KDEy) relying on the Cauchy-Schwarz divergence (CS) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors proposed a Monte Carlo approach for minimizing the divergence. The distribution matching optimization problem comes down to solving: diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index 5de2717..a7283b7 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -1038,6 +1038,10 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): class-conditional distributions of the posterior probabilities returned for the positive and negative validation examples, respectively. The parameters of the mixture thus represent the estimates of the class prevalence values. + This dedicated class is kept for backward compatibility as the historical + HDy implementation. The same historical preset is also available as + :meth:`DMy.HDy`. + :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be the one indicated in `qp.environ['DEFAULT_CLS']` @@ -1273,13 +1277,34 @@ class DMy(AggregativeSoftQuantifier): self.search = search self.n_jobs = n_jobs - # @classmethod - # def HDy(cls, classifier, val_split=5, n_jobs=None): - # from quapy.method.meta import MedianEstimator - # - # hdy = DMy(classifier=classifier, val_split=val_split, search='linear_search', divergence='HD') - # hdy = AggregativeMedianEstimator(hdy, param_grid={'nbins': np.linspace(10, 110, 11).astype(int)}, n_jobs=n_jobs) - # return hdy + @classmethod + def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None): + """ + Historical HDy preset expressed as a configuration of :class:`DMy`. + + This preset reproduces the original HDy setup by using Hellinger + distance, PDF matching, linear search, and a median sweep over + `nbins` in `[10, 20, ..., 110]`. + + :param classifier: a scikit-learn's BaseEstimator, or None + :param fit_classifier: whether to train the learner + :param val_split: validation specification for generating posteriors + :param n_jobs: number of parallel workers + :return: an instance of :class:`AggregativeMedianEstimator` configured + to reproduce the historical HDy preset + """ + base = cls( + classifier=classifier, + fit_classifier=fit_classifier, + val_split=val_split, + nbins=10, + divergence='HD', + cdf=False, + search='linear_search', + n_jobs=n_jobs, + ) + param_grid = {'nbins': np.linspace(10, 110, 11, dtype=int)} + return AggregativeMedianEstimator(base_quantifier=base, param_grid=param_grid, n_jobs=n_jobs) def _get_distributions(self, posteriors): histograms = [] @@ -1703,5 +1728,6 @@ ExpectationMaximizationQuantifier = EMQ SLD = EMQ DistributionMatchingY = DMy HellingerDistanceY = HDy +HistoricalHDy = DMy.HDy MedianSweep = MS MedianSweep2 = MS2 diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index 706d9a9..39e1c21 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -97,7 +97,6 @@ class DMx(BaseQuantifier): return hdx def __get_distributions(self, X): - histograms = [] for feat_idx in range(self.nfeats): feature = X[:, feat_idx] @@ -161,8 +160,6 @@ class DMx(BaseQuantifier): return F.argmin_prevalence(loss, n_classes, method=self.search) - - class ReadMe(BaseQuantifier, WithConfidenceABC): """ ReadMe is a non-aggregative quantification system proposed by @@ -187,8 +184,8 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes. - Of course, this approach is computationally prohibited for large `K`, so the computation is restricted to data - matrices with `K<=25` (although we recommend even smaller values of `K`). A much faster model is "naive", which + Of course, this approach is computationally prohibited for large `K`, so the authors advised against computing it + for matrices with `K>25` (although we recommend even smaller values of `K`). A much faster model is "naive", which considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior). :param bootstrap_trials: int, number of bootstrap trials (default 300) @@ -230,7 +227,6 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): self.rng = np.random.default_rng(self.random_state) self.classes_ = np.unique(y) - Xsize = X.shape[0] # Bootstrap loop @@ -303,14 +299,16 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions ' f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}') - # we convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5) + # we first convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5); + # this will speed up subsequent comparisons a lot K = X.shape[1] - binary_powers = 1 << np.arange(K-1, -1, -1) # (2^K, ..., 32, 16, 8, 4, 2, 1) - X_as_binary_numbers = X @ binary_powers + binary_powers = 1 << np.arange(K-1, -1, -1) # (2^K, ..., 32, 16, 8, 4, 2, 1) + X_as_binary_numbers = X @ binary_powers # e.g., [0 0 1 0 1] @ [16, 8, 4, 2, 1] = 5 # count occurrences and compute probs counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float) probs = counts / counts.sum() + return probs def _multinomial_distribution(self, X): @@ -319,8 +317,6 @@ class ReadMe(BaseQuantifier, WithConfidenceABC): return PX.ravel() - - def _get_features_range(X): feat_ranges = [] ncols = X.shape[1] @@ -334,4 +330,6 @@ def _get_features_range(X): # aliases #--------------------------------------------------------------- -DistributionMatchingX = DMx \ No newline at end of file +HDx = DMx.HDx +DistributionMatchingX = DMx +HellingerDistanceX = HDx \ No newline at end of file diff --git a/quapy/plot.py b/quapy/plot.py index 5c5e4b4..e4d2529 100644 --- a/quapy/plot.py +++ b/quapy/plot.py @@ -1,12 +1,15 @@ from collections import defaultdict -import matplotlib.pyplot as plt -from matplotlib.pyplot import get_cmap -import numpy as np -from matplotlib import cm -from scipy.stats import ttest_ind_from_stats -from matplotlib.ticker import ScalarFormatter import math +from matplotlib import cm +import matplotlib.colors as mcolors +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap, ListedColormap +from matplotlib.pyplot import get_cmap +from matplotlib.ticker import ScalarFormatter +import numpy as np +from scipy.stats import ttest_ind_from_stats + import quapy as qp plt.rcParams['figure.figsize'] = [10, 6] @@ -560,6 +563,123 @@ def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs return fig, ax +def plot_simplex( + point_layers=None, + region_layers=None, + density_function=None, + density_color='#1f77b4', + density_alpha=1.0, + resolution=400, + class_names=None, + title=None, + show_legend=True, + legend_loc='lower center', + legend_bbox_to_anchor=(0.5, -0.08), + legend_ncol=2, + figsize=(6.8, 6.2), + class_name_fontsize=10, + title_fontsize=11, + legend_fontsize=9, + ax=None, + savepath=None): + """ + Plots data on the ternary simplex for three-class quantification problems. + + This utility is convenient for visualising prevalence vectors, posterior triplets, + confidence regions, or any other points that lie on the 2-dimensional probability + simplex. The plot can combine three optional layer types: + + * `point_layers`: scatter layers for one or more prevalence clouds or reference points + * `region_layers`: shaded regions defined by callables on prevalence vectors + * `density_function`: a scalar function evaluated on the simplex and rendered as a heatmap + + Each entry in `point_layers` is a dictionary with a mandatory `points` field + containing an array-like of shape `(n_points, 3)` or `(3,)`. Optional fields are + `label` for the legend and `style` for matplotlib scatter keyword arguments. + + Each entry in `region_layers` is a dictionary with a mandatory `fn` field containing + a callable that receives prevalence vectors and returns region-membership scores. + Optional fields are `label`, `color`, and `alpha`. + + :param point_layers: optional list of point-layer dictionaries + :param region_layers: optional list of region-layer dictionaries + :param density_function: optional callable receiving prevalence vectors and returning + scalar values + :param density_color: color used for the density heatmap + :param density_alpha: opacity for the density heatmap + :param resolution: number of grid steps per axis used for rendering regions and densities + :param class_names: optional list or tuple with the three class names + :param title: optional plot title + :param show_legend: whether to display the legend + :param legend_loc: location string passed to matplotlib for the legend + :param legend_bbox_to_anchor: optional legend anchor box + :param legend_ncol: number of legend columns + :param figsize: figure size used when `ax` is not provided + :param class_name_fontsize: fontsize used for simplex vertex labels + :param title_fontsize: fontsize used for the optional title + :param legend_fontsize: fontsize used for the legend + :param ax: optional matplotlib axes object; if not provided, a new figure is created + :param savepath: path where to save the plot; if not indicated, the plot is shown when + `ax` is not provided + :return: returns `(fig, ax)` matplotlib objects for eventual customisation + """ + if class_names is None: + class_names = ('Y=1', 'Y=2', 'Y=3') + if len(class_names) != 3: + raise ValueError(f'expected exactly 3 class names, got {len(class_names)}') + + own_figure = ax is None + if own_figure: + fig, ax = plt.subplots(figsize=figsize) + else: + fig = ax.figure + + if density_function is not None: + _plot_simplex_density(ax, density_function, resolution, density_color, density_alpha) + + if region_layers: + _plot_simplex_regions(ax, region_layers, resolution) + + if point_layers: + _plot_simplex_points(ax, point_layers) + + simplex_ymax = np.sqrt(3) / 2 + triangle = np.array([ + [0.0, 0.0], + [1.0, 0.0], + [0.5, simplex_ymax], + [0.0, 0.0], + ]) + ax.plot(triangle[:, 0], triangle[:, 1], color='black') + + ax.text(-0.05, -0.05, class_names[0], ha='right', va='top', fontsize=class_name_fontsize) + ax.text(1.05, -0.05, class_names[1], ha='left', va='top', fontsize=class_name_fontsize) + ax.text(0.5, simplex_ymax + 0.05, class_names[2], ha='center', va='bottom', fontsize=class_name_fontsize) + + if title is not None: + ax.set_title(title, fontsize=title_fontsize) + + ax.set_aspect('equal') + ax.set_xlim(-0.1, 1.1) + ax.set_ylim(-0.1, simplex_ymax + 0.1) + ax.axis('off') + + if show_legend: + _, labels = ax.get_legend_handles_labels() + if labels: + ax.legend(loc=legend_loc, bbox_to_anchor=legend_bbox_to_anchor, ncol=legend_ncol, fontsize=legend_fontsize, frameon=False) + + fig.tight_layout(pad=0.6) + + if savepath is not None: + qp.util.create_parent_dir(savepath) + fig.savefig(savepath, bbox_inches='tight') + elif own_figure: + plt.show() + + return fig, ax + + def _merge(method_names, true_prevs, estim_prevs): ndims = true_prevs[0].shape[1] data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))}) @@ -612,6 +732,94 @@ def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error return data +def _simplex_to_cartesian(prevalences): + prevalences = np.asarray(prevalences, dtype=float) + prevalences = np.atleast_2d(prevalences) + if prevalences.shape[1] != 3: + raise ValueError(f'plot_simplex expects prevalence vectors of shape (_, 3); found {prevalences.shape}') + x = prevalences[:, 1] + 0.5 * prevalences[:, 2] + y = prevalences[:, 2] * (np.sqrt(3) / 2) + return x, y + + +def _barycentric_from_xy(x, y): + p3 = 2 * y / np.sqrt(3) + p2 = x - 0.5 * p3 + p1 = 1 - p2 - p3 + return np.stack([p1, p2, p3], axis=-1) + + +def _simplex_mesh(resolution): + simplex_ymax = np.sqrt(3) / 2 + xs = np.linspace(0, 1, resolution) + ys = np.linspace(0, simplex_ymax, resolution) + grid_x, grid_y = np.meshgrid(xs, ys) + pts_bary = _barycentric_from_xy(grid_x, grid_y) + mask = np.all(pts_bary >= 0, axis=-1) + return xs, ys, pts_bary, mask + + +def _evaluate_simplex_function(function, points): + points = np.asarray(points, dtype=float) + try: + values = np.asarray(function(points), dtype=float) + if values.shape == (points.shape[0],): + return values + if values.shape == points.shape[:-1]: + return values.reshape(-1) + except Exception: + pass + return np.asarray([function(point) for point in points], dtype=float) + + +def _region_colormap(color='blue', alpha=0.35): + return ListedColormap([ + (1.0, 1.0, 1.0, 0.0), + (*mcolors.to_rgb(color), alpha), + ]) + + +def _plot_simplex_points(ax, point_layers): + for layer in point_layers: + points = np.asarray(layer['points'], dtype=float) + style = {'s': 25, 'alpha': 0.8} + style.update(layer.get('style', {})) + ax.scatter(*_simplex_to_cartesian(points), label=layer.get('label'), **style) + + +def _plot_simplex_regions(ax, region_layers, resolution): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + + for layer in region_layers: + mask = np.zeros(simplex_mask.shape, dtype=float) + values = _evaluate_simplex_function(layer['fn'], valid_points) + mask[simplex_mask] = values + ax.pcolormesh( + xs, + ys, + mask, + shading='auto', + cmap=_region_colormap(layer.get('color', 'blue'), layer.get('alpha', 0.35)), + ) + if layer.get('label') is not None: + ax.scatter([], [], color=layer.get('color', 'blue'), alpha=layer.get('alpha', 0.35), label=layer['label']) + + +def _plot_simplex_density(ax, density_function, resolution, color, alpha): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + density = np.full(simplex_mask.shape, np.nan, dtype=float) + values = _evaluate_simplex_function(density_function, valid_points) + min_v, max_v = np.min(values), np.max(values) + if max_v > min_v: + values = (values - min_v) / (max_v - min_v) + density[simplex_mask] = values + + cmap = LinearSegmentedColormap.from_list('simplex_density', ['white', color]) + ax.pcolormesh(xs, ys, density, shading='auto', cmap=cmap, alpha=alpha) + + def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None): posteriors = prob_classifier.predict_proba(X) assert posteriors.ndim==2, 'calibration plot only works for binary problems' diff --git a/quapy/tests/test_methods.py b/quapy/tests/test_methods.py index 91d23d5..54954a0 100644 --- a/quapy/tests/test_methods.py +++ b/quapy/tests/test_methods.py @@ -7,7 +7,7 @@ import numpy as np from sklearn.linear_model import LogisticRegression from quapy.method import AGGREGATIVE_METHODS, BINARY_METHODS, NON_AGGREGATIVE_METHODS -from quapy.method.non_aggregative import DMx +from quapy.method.non_aggregative import DMx, HDx from quapy.method.aggregative import ACC, DMy, KDEyCS, RLLS from quapy.method.meta import Ensemble from quapy.functional import check_prevalence_vector @@ -175,5 +175,19 @@ class TestMethods(unittest.TestCase): self.assertEqual(len(estim_prevalences), len(np.unique(y_train))) + def test_historical_distribution_matching_presets(self): + dataset = TestMethods.tiny_dataset_binary + + hdy = DMy.HDy(LogisticRegression(max_iter=2000), val_split=3) + hdy.fit(*dataset.training.Xy) + prev_hdy = hdy.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(prev_hdy)) + + hdx = HDx() + hdx.fit(*dataset.training.Xy) + prev_hdx = hdx.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(prev_hdx)) + + if __name__ == '__main__': unittest.main() diff --git a/quapy/util.py b/quapy/util.py index 94b75a5..a774816 100644 --- a/quapy/util.py +++ b/quapy/util.py @@ -152,7 +152,7 @@ def download_file(url, archive_filename): def download_file_if_not_exists(url, archive_filename): """ - Dowloads a function (using :meth:`download_file`) if the file does not exist. + Downloads a file (using :meth:`download_file`) if the file does not exist. :param url: the url :param archive_filename: destination filename From f6c822ca8fb121b41dded78ac766e8e29727be9c Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Fri, 3 Jul 2026 17:07:02 +0200 Subject: [PATCH 28/43] improving docs and new example for image datasets --- README.md | 2 +- docs/source/_static/custom.css | 53 ++++++++++++ docs/source/_static/quapy_logo.png | Bin 0 -> 18016 bytes docs/source/_static/quapy_logo_dark.png | Bin 0 -> 17920 bytes examples/19.visualizing_simplex.py | 68 +++++++++++++++ examples/20.cifar10_quantification.py | 64 ++++++++++++++ quapy/data/datasets.py | 2 - quapy/method/_helper.py | 107 ++++++++++++++++++++++++ quapy/tests/test_functional.py | 27 ++++++ quapy/tests/test_plot.py | 41 +++++++++ 10 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/_static/quapy_logo.png create mode 100644 docs/source/_static/quapy_logo_dark.png create mode 100644 examples/19.visualizing_simplex.py create mode 100644 examples/20.cifar10_quantification.py create mode 100644 quapy/method/_helper.py create mode 100644 quapy/tests/test_functional.py create mode 100644 quapy/tests/test_plot.py diff --git a/README.md b/README.md index 730a433..fcb820d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ for facilitating the analysis and interpretation of the experimental results. ### Last updates: -* Version 0.2.0 is released! major changes can be consulted [here](CHANGE_LOG.txt). +* Version 0.2.1 is released! major changes can be consulted [here](CHANGE_LOG.txt). * The developer API documentation is available [here](https://hlt-isti.github.io/QuaPy/index.html) * Manuals are available [here](https://hlt-isti.github.io/QuaPy/manuals.html) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000..600ce46 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,53 @@ +.navbar-brand.logo .title.logo__title { + display: none; +} + +.navbar-brand img { + max-height: 2.2rem; + width: auto; +} + +.navbar-brand.logo .title.logo__title { + font-size: 1.05rem; + line-height: 1.15; + white-space: nowrap; +} + +@media (max-width: 1200px) { + .bd-header .navbar-header-items { + min-width: 0; + } + + .bd-header .navbar-header-items__center { + overflow-x: auto; + } + + .bd-header .bd-navbar-elements { + flex-wrap: nowrap; + } +} + +.hero-copy { + font-size: 1.15rem; + line-height: 1.7; + max-width: 56rem; + margin: 0 0 1.5rem 0; +} + +.landing-grid { + margin: 1.2rem 0 2rem 0; +} + +.landing-card { + border-radius: 1rem; + border: 1px solid var(--pst-color-border, #d0d7de); + box-shadow: 0 10px 24px rgba(15, 23, 42, 0.08); +} + +.landing-card .sd-card-title { + font-size: 1.1rem; +} + +[data-theme="dark"] .landing-card { + box-shadow: 0 10px 24px rgba(0, 0, 0, 0.22); +} diff --git a/docs/source/_static/quapy_logo.png b/docs/source/_static/quapy_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..379899384b6284b071ce071d8b5f41738e8a3c0a GIT binary patch literal 18016 zcmXtg1yoy2*L6a0cP-Z9P~6?6P@p&zcX!udrBK|pXmPjV1b3&nySos^sR97N_(GriAi_iM&ph-7pbxMv;&N(;(BX?{ z`W^Z=lB2A)3jo0U;olAD`Aftd`XZ65l$NWigSo4Rk+T`V!^4Bw%HGDs#K_T%*}>T& z<4lkk0H6TKNs6g?W}aqw`Ks;SzKQHNbEJh(+veq+P|(rkVa031I1!ZnDXr0{K-Q{e zP`{&kuo*bl#8Jlg_z}9Hu`GW2yMei>p@fP9NK&C;v#JBzixx*6GU}Xt{q!M+N&Afd zo~HS{gCSjT!V6v7d|YUPFI(c`|Au99F=8kV3E|*O(`J|F=f1JVmf1i$Iyxc+?n?*bhm5-k$_zKn=#ue3IgVBoQ9)cumm&LN*o5$bi z*D}8SI!lrgAa$Pn-*Iuf42K*My%9u3x04NeZ5U9^ehPx~@+>DL87|qtYw6sWA_P_X zo`F9(+EBC6udef(LGWDqz`PpC6@@@*EJQy)D`EjhNQhnrPt_CqwZ8X%QO65YGJ)B_Teu^#H{V2+6#Gwg(sIUF+CI0 zDN2n8JJ@7dXC#sd$?J>1X~*16dDwr05i#^re_#%uPcKC8cwWY1@le&mP-*_g4 z@>A5~sOB8bmlzQ~zkLQ<6YI4*m{$sZY8mqMJ& zJNX~?tkKScJMrkgy-I}Ec5EWn5zaM zxiQxdz{fA#=VZOQavzl@+k$3r_!}b`zb9t?GRq~nXb|@|^-o2T_{h4e`n-kU00Zk)&;52vWX1<$tpOf|QuwOPnVSJXGs7AaP<%#u1f zojChNk9_~%HxF`lB(kH3HqPBLJ?N9H9@fv!u3QR(Q(SHNSg_LqoWK5P16L8{O%-31 z`2yqm49E*rC34|{nHmoN3uU8tLq~ITQIV;d-P5I`P<8jmnMKMyGw->w@|DRsVwMe? zZz=kpc0j%bq>a=i2@yXueg1}#{TpXuNV=#wMRU8T9qZB#{6aMg-6>)>KkrDNd!P|k zwy2%0UBpkkdC_);{SLV73do;G{^<4yFy_wMm_nyxh;X;Dl!dLWZ9?GKF9@#<{p6$8 z89x|*NLre1899(*bvFBNZdeiY%1oS0O5^4i_974ftdJN`@nlnnrU}ENAE4~3&IX)s zSgbjJFxb4%QtNw;*^mzW8m`t|^Z-mk z0*J8!fOMtEc(C7XVS9eb!Em#0cEUb~rM;qT-RHm?UwXQ>)&&S)V8ld=^9r654o#|> zyhhUY8H-p^`k$)qW=;B%jLdhVJk)>D(wbxW^=X;wIyEjy;VcVrgRgbsYVaK#9Ca|I zCH8j-SmYvV3?NrjnYju5HWR!wm@VjD8wkQgLjNYe{y|wGsO$x1qUx(=uFdzxFqu}9 z-4z$nE2E-SUcxn?G7N+a>=8^4+_E!VLJW)nKC4B6GH=ltxN`8`o$IvB;2=W~gH#bM!3YD8 z>4%d5XVzy4`jFo-m)rZOKoiG)bkn?oVH|*9Rt<*`(|ZQ-+;#%?@UK7x*t%0JeG)oA zw;_5Fkq2ClS-|@}vl=a_6{Ai;LBaF`jBY%D9uQER`9dvfBc>I4jQ@sa`t^@z25c}& zKd8mNQKR#s`vqzF-4*Y5N%6|@Vf=fsd|h;`N+}dz_WZ9_jLmDSsGN_%;O%y#cc)=; z3FfdtakVx zNbb3YLicZqVf7Af(aYt(jwC`|{4u7%KP zt1R!Zx*QK@6mA%RcG2Zl7?_UyurES{%zhLsEK7Y>0sq!3xCM}kKq(BJFDv(-1>()h z6W%m;>0IwNPE+vDfjR9o8Ons{6UY7sVFG@T6!SlqhCkr4F~bvYoR=_SXprAiTbTW- zeFbh{7x~oyfg6S8C%qyl$nrgk0{`aeeLH|jX1Cm6`;t%Zloy=FWv~w;Yw-jMsOiDL zK*PL&UDh6TO+At=LVvir9*M@@3E~c*`NToMdn#>VFRlkHkaxP z9%{<72*px~0fM053YuOxXVpN}9%{|BH7hSiSUT}UeQif6Lrc#ma>06u__-tP2`co~ zxzpKIA12;|WPJH#UawI;7yadwmfJzrI9a(W($)hE3OyF^DJi+=8xAV1k{q{mRb|-^ zZr;<_urQgr$Zs(=)y{KN3VMH&@!-Rq#|^8ar_n+4%}Fjwa62Db?)`kIc=80Yqek?E z&7F7K;wb)D-d`c4*6s{9;`bTvI>7b($_5@34KZx`g!vcG!23QvaySF;UB6<-#kUUx zFo4|@pQac8A)j(_!SllGu+eUj3kzD~$H!}COS#-tC`vFQd=XU(#j6{K+{&+W+9)Cm zhq)HNu}#UhV5IrpXJ~P?GJ?0=&fx&>4SzRbLq177alrj@fTx?7)ZbIr(S-3t0ym}& zM80+z9$sF0_I2I|6wm1IBF1q<59$R30ENe96h@dH-ZY zssW^eg1U-Ih~8w3<{0cY-wC~s=6_=x+)u6FlEc7XgIexMO@ke{8`kYL#*fh_x!^s{ z#VKw>_WpQhnBxi`_?(o2X;pW2KVQpZn`GT>vl8kNPy5ISKo^&iFf$IU(`(vh8 zR3)+VThH$(*Ci}yKF8-Hh6!Mw8N`C(fIzZaptS8+I#+3K&vbJcz?P@}cX*8!-*UTm z0~IZ3kgGn$3a+vvAZ2dXDp&289fQA^Cr1_Dwab0_dT?`>lxm_*E=3l5^d}wGMN!C|B-Yi6mq`>SyTT6g)tu>f=}b z^(<{+c%W~7*D5x$P=Q6yan))w5yV1g9>-A`GEZNH5jtGN*8EFv+Ys@9Bz%-yORwEB zVWz`XtzW6h?t9`9qv0}#4+-#D12FOQ5;$L0nis`?H%InB?&WpuV`}lE`cqRXVG3kT z)UDbesR|cAS$eLb>|wayMNp7^2wg$jxFYqk)RH^#uP)Ue-3Lhw*&~b#S+F7uFU$ge z_KSCU`ZX#I%WK_Q6vKz&ZRD-ngw>(WiQF=K-G^UP1PiR2opd{2Uy zsB7OZvYl2}$w?OOJ;3#cwCvLRyJP1y4uW6zQ6BnZvN=M+qaBxG=49)5(ZB>3i=z7a z^*{1xUJ)V8HK(}20ko-P5q%8SK$~xQf|4f=bF zK%N7Cahj+I>Dm6Ooo#N`WFM!reWRBjhrKofFd^pLiE-j6Su0`U;wGZPJJy>&p}NdB zKc&^4H~L?z(+4beAN_%W}RN%kXY8PnFPsL|U9=Hj3YnnF3%%$Oh!TBbDw*nx*S1PR%$^lGeWZR0|NslJmd{@hNX)|41)3uq8G zc#i-OoDz``KUs`D!Ug#l$wTY8M-dVo6eSx@PklRMg_cxH4QOd816pi#JY-pX#4)4q zqVPB>K03OiV>6J!7;d@&I$7ZGdX4cqYsU-uj*iIW`fM2Xf-d}06U2BE*AzNhKi2PW zPBlLc>3xk&;wJ?&r$ymYzRIZ%ST0vbV6`a7x7m%Bm@g#!MFX=aXwgAR7q`ou2Ur&u z=V1uGOqN2@EDW42TaApY5NOn%FBr z04QgxRlgLkjyte=uGC3bQ5ln=6~7nfBj@9Nfe2SgrS%Xuik*&hTg%pBP>>ha6+KH} zg|%WXd6+nLOrJ5z=RXut8P$fOqNPi_6zE`W?z!2OhCic^8p>+8r^$*u#N(c4s~Im| zGy@A!WTaz9a{^v%dB(Un_XsWln75$G7%;QKAk4PFXP#~{V=ADn#f1gH&g_C#;aM@7 zWuMgHXQ`B~-y3szsuxzscc8R@Q0FeM+BiB$ehHldE_&VtA@*(tf##V%HG4H%boXQz z;BUVGzfD6pQK=1@LWB1x1-Go+JVwUUhu#AK)@lhK0BuuNFv9pHg436H&LCLgL`__w zyh;XZ1H&3`b9DD9uGrc~Y52TAK%voYd4M6UNVqn0wv{Jhmtj2*ol{CE%^UUw0&|3> z`s`QB8E0#obMLC0oPto-h1H7Sa)V@ZNHeP$j`O zQlrerAYja)0yiRtOfr4H=OJnY&(n8YP`B=3q%ma)8Zs8aPI)zBHwX$H7Wvu)-ybCO zjM;TpV)C3|3>d5ouKP>B`WW+!0pq?^suTavWg{8cCy_T12(@Ga zPR?V04MFp1DeN#%$8&S5{++$F6iQ;IBa{UmOC~fG&XuX^KW3`gH6rTT7|k>^I_!g< zL_YLpfqm4OX2l261QNp41@>;DnqAz&{;k*=m@{lTiGkakmcP0x)b>-{H$0mD4PamH zLI$kQ)JMoEabKEHSXUf8asB9owTwbS0T!_RVGp?JrYX=$jrvsc4z^~N;6f}CmR}An z8*{~9Kwmq$W;gU^h^RTX<127_twQPmA>c25xqT^uqrks4HRQeZXB{PvSI~R7li#yu zVITW@>tRPZC})|DYDjFJem>^o4yT4m&&NneuLp{2N5g&#E6hDw3dpXYMA`U(=*iF6 zI`ITOr5m{fU8<}pec8+R02*?tM^CkJ%iGPyL#{6|mI8;!Os)%h=Jq0QxL*+T!!%a; z2_cX09-ZMO#FZ;tIdl<~=RaUQqQO8Qt*9-?bFSy;p9>N+bqoi30RLuP zxnQT?u;4_`*qbPF1mNnlRON>Xyci*EZE4rUWX{;B=Qxcxly3}t@Sv%{q?`QKq5o;# zCw#3;Voag=35|6=wVD+W>>jYr9#H?@kHgSc=R2OZE&$Tvwwn!HsBjZvYy)bH1-+r1Z*h0tpg+x@RnR8T9_w9s(2L-GC_4^;KO4o{|@wzb~a<5g>X( zuuVpPfoX{n8sRNDS?jhRRkQg zLB&?UuFQ@UCae_xUZHw(Z%XC$FGE7Af6xI zU2_o;VJh?OqsDd-orL^pa!=b0#s`&&^TE%T`8N(qfLd(13jZ8J`DkrD$ zVQn$M9CrW!+>0p4J>#`|4Kj=u;kFB@Hk*$IHy0_TrmsH;z96+T=2D8u&$O(yiCDH9+%Rk)!yfYaja zOY0>gc9laW2`)-vNqKsHA6{>M=&C%_uM6QMv0ZEOQPc!8*Dhq8D<5JrxhXYMV!Q7avaEqo6}?|g@2e(d@`b3mHVO*!)Kv5-k5`^GvI^syJ>+5^l;yHdo-KkIf`WEd*>^j_t#vdZoGPn%r|yU_ei3&f z3I$9QlR0f|i+?kCytjP}7mBh5W=VxYxu=veppDaGgad4RyXaGBlGf$;Fpmyf-CK?Z zPt$ES|452v`~_arZ3zQ~ujZKSjZ{%!Q%9CYLC5m8D??868w&@!nriYh3WSk{3AHCj zZ(nhC~$Hzq>(YGXKShrDB11gnil*lgV`5RtMXrXlDCzwa0hdFR?(~L>r)Mz~l!1{Mvi7@T$89MqS zF#N8;q;dJJ5`tf4_#+~3(a`c|qNFznnL{=qK|1v$&Qe>51A$fWj|pRoTfovM5A+bZ zDS}uIe)DfE^dILcT(FpWQ@6JMEM;^14ohuB4xZD=$~{#^HFR@M)uS1OlO>OYe`3ve z^7c>d>Bv{E40^jpb~>@dEe;_HhArekv9<39CCuCyJ| z2TEOg18wE#i^J#Fi#SqyNfl0FJeH1B!k)yyX%MZU>tfsEDHdSO#^p33ttSwbyAqL~ zs56}dRz&-GJu=IS*Ss#xpm!c_PJ1hSgc4tgt%iP)XowEhuzd$S@LNrjWeGPTQMfl! zo4?TxW=4m`Kh89V0g6s}oZWc^t6HhnLk)i-Z=inQP7fPL3EPjUEYZHHbmJC)5u7au z#FwEfls%-^dU|?~)0lQ6B>*c;ErfXS^6jJSGcH7daTNyjhX8F(X(P9uy1JAHw6{lZ znVB>uT7{mn>2ulFi%4@eDb*!OZ%3p+8Ht}xED@A*(MKe74S;5IJV4K9BFgeQU#%wj z@V?>o_pv1eZtMmuhq-~S*t->a^QZY_qe;Ea#%+bhj)EG(?Tv+77}CK)fZ~#78;`b@ z=MpIYlE#E-wDB>&5j{{1`?r>98Ayz#HMO}f8!N8RR~?CZebT(jO1O(Cn3`P_qtB4A7+Nb=vACOh+`UcK*jyOIc{p{>2TKF^`7N zK=qG=5Js!lRip$=L5i}Y-=R{BBw~F_dyGCsGl$r6U-QZfURoxhkuY?a_v?N1{cB?$ zEe;RgXM$4HZ~Q|)7;QDF;h4c%CCVTMbdiI1_x-X5+8op>Mp+s!$(;IKAxC7_J)5_d zfvGY-pZzcD%a^2bWuQlvkq)tA#H8-{_wYh`q(%1)SKYrd{;?t z3yknwicl@Q%6V`}%a;}++dyZC2}cZE9b49yZkr~1xhyMkPRJxqqRRmCLoEr(Kv1*l zpd?~R<~ut%pM$Bid`d4zb0Plkxr(NbTF5g%@{&M|ii=;R4e0PulO=#JIpLig9>Bm70i=w;M~n$X|DiwfL@NciJC-pyfUpT8=9?QkYgIT*PB zrp;4k`A$f)j#?^3cbD?*k-t}KK_a%))9Gi%yx$G8BC^!oUqrI+)Mekw;}7i?z2JqUpA0!fqtzVPHHb65a7 zU0-*`h1|V`LRJ;R^sUelFHwgfD;vzlnc`EN+~B@In|=*_oq6NTh@hlIwY7cWaqDP= zyhD9=_+4)_X~$c)X?qInBJI(s6DD-`acwJX8%5)fb90-5$Nb1@7Q$9KqqJr?DV#>^ z@>_M3gjHy1a=J*)0zw;Ng=J}?psx$3Iwp4GSKi`Qy?MF{qY1-WNI$X#l+3s4u{ai1 z(#$fG8NP6PHjk1pq7Q=&4nvJjKJj&T5K%FoC4k956LiYmFY6n*( zU|*P3aMN#8>~PX$oi#jBkY#ZZK^6;7F&omg-fh zAYcD$GxWa)zC<0e1qB2i@#axImjU<2+S=^66g+O9$zsAj3tAn9P>z}*e+f+v75~ja6f>qS?UL7=4FPh`?0c8+(OO!A#+Y3@@3;z8{a|UgVDk zX(~tB_AZY&O2`F*W%j>88-xqFTS&TJFB@vwr4K_iTU53Z$ETsF1)Bf!@(zT7r#sE# z-ZQAc-Ao12KGu@X!EUv}GEx(UjfCme>C;if*fhRkig-}@)i1po<2icF;W|r`*9z_f zF5S+Pd~W>17Hg{NsmGeV$Hf;Ufcah;Ik5Q{d9}5GletgaTDm|B{o5^Pa>246L{qm- zC(cysUCm3+K|B-+4-9(^fGz81&fdXa6F!$JOhD{=Fb`wn9Ogv#eJufoM{Z!mEb-!{ ze6RS|8xr$7jxW$|mBv^zX3#54wN$+H)n=nc>^P&2c|v-jxm*cGmqHQi)P!q=0pB?x zAcauy6i-3_PAS+Ys%KI)WpUvQ)S)EuGJs^5O2WH39@pz-cxx+*audcBhWIJ}@h|R2 z*0k36-DsiQUw8^;2dPuqRoy}W)U(!0Mu_L6l7fuZL3FEEc|8Wh{xkMy0LofVa@tyKb$dsLV*<>-^oDfEB_d8-J%@w^jO zt;upiS8)Cy{>EYZ!c&GXq|N5XH(U!q8 z39Z;6Nbr-^w%n6ZHr2Nh9^G#jFdd5M66tgrMbTaos(dbV8sljZNiOGJzYgh^1wUsH$3k2hb#BGmP=FiK1B zt20g!^=7nT$7>N*=ihLZo(rQ7m%sdQLw)4nghEQ`%8Q#}9{{OyEYyX}{Q*|&$(yPd z2ev;PLYio)9@DKai&+RIC_Je|X#=M)OpoP<^oyPOWw(!$Vn}Xv+v;4!AP6~!Es6C8 zV`P!!@s$IwNL(MqOx4Q0c;Y)`(9XEfj*%X~lC5YD5m3YxHPlY4lSbqaBaA&UKnno23lLIYFEeDSl~6)(sViI08{SRmdXWuFq1tXR z>T;Rjvo_PBD;pQf|6W*Z$9AUQpL;lOh$9~MjiGJ0f@5P<|&o&G~CH6{L4gA|A@-awA9FhgaM;PABbPd z^giuS^@V*tH}C(V?gfKRro5hO)J<2%J?+!{kq;4rB#qNb+HilIc|Wi>hLD#)R0#&K zRd^4RFm_C4AM&$zmb%4Mv_K69lRxN7kUyz_z<22xExsp8Oa|0bi<_RGf8_zsjiD1H zNybK^PNEUJ6d0Tq52;P7a~D`;(>c7x14d7*XH|bYpS~qXLOKbAGH*)MQ@sr#=q5@k z#1!wZ1H*D+;VSDtSO&>~4CemuJ%lWW6Y&^zfeo8(0-=?=-#FFlAOyomZ@(zaX45Ul zUYZ!yHA!D~7C#Zvikz&Z|4c9*XXJ&H!?z*K|cHa*PVGIHZ$L!^iU@g4tkJ3yWj zH(w+-&5wEfIu5~N?PBz!kpUcNUkLopZzEZj%pfu<(H=_(MT^3zF`wM{eBI0A3w5ba zd^2<_{m07c^Ay}{APhA8nsnbor4i~f{VDA>#VyjEBCu?;qbE+W53X< zxFgFrnlqCE>A}juf?@ z*s%sz4(&9PRT}bqFESpEFL~MvsBDEY>NK=EJEDahBL{3Gc5@4X?NLgp7d`K_ z42+A~%^tcpy;il;&lzmbZOIwJ8fqjT?qg8`v@s~CCu)0a-!tCwq^gBe^xqG}u$)e6 zC$8v6rNA86O^*hc)Ch>wbZ6he%CU2$Ns@o5fmZ((fDSqr3-6|o6tN>h1$`Z>G^u_9 zehrV2ksmGcxIS0Y`v5|RL_@C+8U&2RnLZ3)7!zqrID79wry8$m(%6x^q)quA4`M7v z#ivawAMc?vA0Ow8hz)HtUTkdp*Vmts1N&P3x4N1QrX+~xshl%XmlEq6r|$48QJ1v4QfvTNs;%?Y3Q!O(@+Cw6Z8QAMqC7{g+xUsfW+qD?CsYa%AY zwQq#N%b&E?lPVLh81Lu2NX75%?IfuIu^ z#-!9QjBT4^`m&S`NdnkrpCKWg>OF4{&Cp0R`P%&3I@Lj=vzR)_9B5OQ=tXn$9*?zp zu)b54YUO@b=MUq;(x+wHyR7(2RaSB;6lH-v)UA+d@US@UPJtrA1p;H#+#K@?VPs-& z*PdizIIDwi!A&^YA3Soc-He$KaWhReUgJ4*9Dh~K`|qzCQPl&Mq13^8SQo%M1mCWW zpi404`bWhZvea{R)rXgFXm|!+H+1Tt%AK*8iYv?L3cxG;Uc<$YBB!w}8nI3KJ8}ha z|6-dPl6)yHM2on4m|;|{d(nbW569K5P9tY{C0n#^Cw>hF-}_a|e`mqcp7;)=TnSAX za;1zgw=%M;&Yaq4TL-2DhjU;IL&=MWOoC3osk&jNZ6ugc#j!z^)N|m4P%0V0k>Eel zw0Z-X@s*|^!hmFHkX7~M1OqBl+rB(-$Kh6B^Y6Bk$SLhFcy~jw5b-++0j8roOI}4FC1?snyE}9?FRw}Hz&Z=$Zo1i(*y?24OmlSnyw-zC2mk(Jak#>ipe~W_gRPHtQ9e*LS;1oo$s+ty&dmn^tu^!_1v?CHb>=wVeFZ2s zzA*K(unAeW%>7{~z#TaBxhS{?Ht95i05~`b^CS%(9q;JFKJ|GzGOM}X^0fT@B#6-N zi&Md!ZoBgYIH`uHHO4*{{nPa|LNxl3!U!_^dpqm9@j(uMG}a%>r(Pr%jE{Wt-g7*S zVIr?`M*^0Ms*9PF*v!D5ujyPi_3O)Ct_+^*8J*J6qazvT$xDkpMlB+sq5`9-<05s| zno7~k@Ky4Q-F+5bBz^aZ&SOF=Zw4tto@E0MQ)DlA4Q+v*#!CAsMRP4TJb{JmE1|Dy zxMMj7m0s`T;&ZXD_);^|TFJYbKzn=I>6vb$76R1pBL!=bsBPxhmgV862Dj(;HTfLE zLMu@V9_>YV0R&PBxTzmb83}{q7o&3 zf!E7Pb5#i5^gRwzxqe7$77P3+=N_@%um@!87^4izQ{hbD<4O7G7){t;(XO~>4&SgF zPGP!kX))omE`PCO-;nrSp1IRUV|P%PlVi#Xj?_h9ues!xmyVKHY#;xaJNc8suQOQO z?b5Xj$`=9-H1Ei|?-y;>0g|)_TJ1KgEmso&0t8>1`7@EaNc_9>1OmK#W6XriGut@xK3@0@h{iej}T$fo*MN9j8h}AASZ^|tO zXHKsAu(bqXAvD?){vg%l{*xNJ$h^E_4DjApfbhoC>G1z(qKmM9yHN-KTKdVFe@H}T zIsDAE>S=H@iZ`Q$mL0c#yCWKeMa$B;{(`=tt6OE{7`=jmkUs4_WtpM6c~!WNam`gg zDp#$Nidg~?32f#&vX2 z4N-FY^tvCWaIFCkETkGhSc#>xro^5mh{aDPjHK53wysBX!NpB;1w7oN0WHh9Ng2*h zO_!dG{)Rm|B&;Ue9Yf9ELM?CXP*ha+<>@jFJD_}r60RA%IlX)hCH?RdGjfaRmN*C* z{E!tGjlPVn33J0g2QGiC>#v_{L()&rHZXr$D01`^1L7!*o^ggCef}ie&(E_&(^+kc z%2qkPlSa0;K%1&xWHLf;?-gZ(l`L;p{g+zu%$npRJ;d!ZvQf1UsDsU9&y`xdXnytS z+0!lLCT>^hmR1ixpoc-_v6cIVst*#k>()$fIJn2r6w6XEj^s2KRG?NTf~S@IbrR(#!( zw<`krV>8cQ!xi2+E@9AGopk9P#q0#SFHS7=yo9)3=2mRuocY@(1N(=Xg1^Q_KS_ta1Z_yRc~9GIA%Mf0S1fASY#MVKVzAiDG~p zce+$-J$pv|@!KqK4lI+^UpYac@52$k>v5>hO3<=Xy9)Iljxq3c)2{ksq|Q7(^!Trs~Q8s0j}yth#r^H7%TR z4||V(ZlBT#h4|BxGBN~7#n*Z_PzJ5SvARe`6;7x&cFx+dJwQ!(_!Hpq2^N7*bZtS4 zui+oa?rQw~9tog4wKO1@%g(T=)d>X1o8r_M^CyfH_`Hlg7?M+Gj~zZ+GC4Ik3% z!v$SCqeViET|m)`Bd*v@-(~4s98clUq8#Y>sa9K$(dgGugdF&aZ-Cj%1?v^g5st%AzBlV z0N4>b$p`*l48*lHWP_r{*c!}q_q>NZ86`%8QSum?0wLt)15Qh zk1o$x^W(AZy7f{E5g!^!dk_ZrjV)h0zLS1e@NwhAM!`Q#P|bpIt5oz@?y!OQBn)_T z_O74RYv?%n(Uel*Xt7}1?Gq(|`NP#!o}MhXdCpnysxOx=x`Nwg_x&C$+U>V?f zto(2po!;%`=Jw+xyM45tu-6@x>Kv)x?4N-~0R~m{%O5`}ChX@^&G~5yBzaFL`ecWy zjM2bE<|nKVe^fcSCUmwaR20#`|Bt_g#pGLvQ@90NyH z5Bv1Wq8kdsGO>y!^ZkOaALvjTZU1$-(`CF2>ZJe;6hQ%;DcA;gdIk4FE!xA z^O}LGk`a42;_v8y;oUN%9i@wo|xeCeE zTr0fK=o4Zzyl*_rzKH>ww}sSg!qqO)VM5m32Ol0ZWwqZ&=^=Oj%PKX zVjT2b2bdL-3-$4L$Q&lG3+-dXxyp+}VE0qqWY+~Z<_%@2@X^2>H(|5hCd_CbaMt=n z$Jc8ma`!>ab!?IPufVAcV`g;H@5uV6SWn+65lRvih%XDj{+uz-cl)RThH?LY_6mX1 zQU8Bhzag~^X8;#eTa7G-sRwmXr49ii6T%_40R7>Ov=)lV2(9w&bQ+u(RGcJ*uzBwW zab7}Q!=l6_CoM^%HkMpu)szmiT%P3akdfPR0_9b#3%Fnomz+WQ&7f~sv{0zEagDRW zqu$#|AY;yzlQQGaiGj7z3Xid7>67(Eu*23tu<=3zxGM>sTkz9|k2MsGyDO6EEgLxx zMY)xotBN^0O8Xi>gR?>?cmjVZb$N6(z~Sepr&n(@9}Nkmcov?q_zj0e7X8Qcv&Rw! z1`g|Mu}K3(XjSzXL4J8ADKKsDOisr(mj@pQ4`5JY(vzaC%{Qi~GUNy$z!ROrK(t4r zV-KNNa*gngINC?WFq@aYVXR(dNX5 zYc93=xT-Xp9dsuH+n^?m@TH>&dPz=%(Y;uscfE z)h$U07K6&6H;15Jpt>q7BD4qO?je`7K`zZy#Tqw-zfjS#d!0`EL2cHzADryWIQeMA zp-FY@{7Y7TpC3mWxbRGQ~gmX4%|ULEWe zz_AbNzh@r0LO`CYDtkx?_6Jy&vPG$<$b_>2N6Gp3kvcKxk)iSJ;4zMXQMN$v?tosE zml~IWqj03@Xs%24TBbPjnnU6#qsO509fWQh-jjZ?y=tFIPIFI1w9ykKt_I8^!Y%@n zoPOia&%XSvbl{9PISd-AUokh%uh7s4rhjRkNAMCPxp2H**LQe-aTC;VqGXE#R%63H z(W)GnFoHv4jb+;kpRb^EeQS5-NPqdz9b;|7&@lk{W!M(im46STHO>$dq0XW^J3UjY z3Kg_oni@Y@Vy>c6B;;2%;!K^!hO2H}3S_jiUs)E(YM0mu)yyH0BJ^~;!A+8G_|#aB zpVv75+>;LJu?wtr9>)ep8SKJj=LkQBgugVZKx_jee|GD$nlwy@a(g3Qh;$At;UPeKO8AS0E5diBEezk1LU0^+AQ446m3m>N{B1o~)XAaxlH< z!bW?`@vl1&=rPb58c=waT%)c2Q|p$EkfT0;dH^eccY+)<#;+)b<`8m1+%o2v{_T;w zCC}tA&`B$Ba>Z*LqCV*p+G2gS5Nth_$^utHp6h%E-8w6j5E`AO4a(DBzG6;HH=H~A z2STe+R;wQF_@Q1nfVZ`t62zp!Z|&~nGo2c{A~AvZ3F`@8kc$XOezZv+>Gnzedo|b7 z#?SBO9bttNdrN0t-@*1^e0qb*kVNQrNj&} zTo5ABPRb##ib?9Uz2-x}LZ2YlQ>{B#RrpEt!1G-@5&d*Y@CDHZ}W=; z>Bi%muRikS%3rr#9qNcRs)MSRq>TU@ok7{4izxl4b@2rwPX`O|!57x`e5czS$0kk{ zRn;2BI=^HTC(n3#1c>c}6Nq775Uu-sQ|TE^b3Ndpha>o6Vf0Jv7BT18@M*>4$(|R& zDtEKaTS9o{+!vul1yky5YrDXT(DTQ*M=X{)gpo;=T2`p*wPCofv{2s>2iW`}bBcUn>LW6M?`~3OQ;A8#b z`&`*Y(ow+-Bq{qQ_VJO7Jx%Aos92HPa8phyc`V>F$rZeuLO{qRn(Ucrzhdk|EL>+o zKCH0gvRj3lq!iUOFK?eQp%^+&4iNR|bL=AO6doY8=_jF2suc#J0b)Ob_~3nMSsjme z6k!(B=qCR}=Wh$g*`dVdj}xPAc}(W#I+qq(;MQM(ynzEC0iFQpNH+8Vir{El&5UYZ z0@ZW(ij^;P^@f-U&xpi0E~wrujfHK0jtw$kI*REKX(w*2@4$caoA#4y*w@4Pnl*Z0 z!oRIO#~r!kLKs|x)NaKTIz19*D(j`7vKaZKn}|80WMV6d!V-66dID(a_*Iy+5mxf| z$==!(0RQ-Rw0@2TJ;6wb%Fgc77n!67 z7&IwSM%%Uz2k>g+f+*Z6jHzZl-y*8QRV|=)hfl50dPBoKyAvOJw8mj`G1IP4%y17x z-p(|o>}fNlITOdn`Q@ITe7m_S#sHE6i&B*J$s^GEktII(jq5zkOg8qJt#IWnX(qb< zTL`_z5<^c-+g3C6$P_#4Jjyew-NaZYw-u)Sp+M zqYg^`)`a0T6<5X}=HNIwHHlR6@B5XPIEQ*VxY(|}Tjt1l)9y6+q(l{)^x+gfKF10K zFP$g`l5ntJ(=|D;!9-`_wSzzg^8xGdIX?2oh=g2*8?ZHI*DaB=m{1q@!axoXN7!sb zCu+o{R%N*GV@6qqgCW1jxw&QM`*B#2fy7^bsVvzQoSbTEZ9L9ta~kMF586H5Ht=&w z$41K!!BqcD8YQkY6fR>+*6XZ(T#+#^-l^|ae`at*#*Xh*5xdl=BWo%E9?-W|WoKqm z8+TL)Q+EIWxW0e?7r?DZ&nk*-fAnz4E#?iQx+ud>0Q|}LZ$mQktP>7C>P5+wFCYQ9 zbIcn?^qjwx9@@!qR16JP0XoE#UL>^~JFZUhDs4{~^mb%_XKCj6z!<Gu2w=w3ACI#S5E!N?Ht7dyP2R`mPjsBitxcG9@4mtCx7gv;u7 zp~a;|@F^jD*`ckiyz72df%yLml>}=0?)v3oybDFVjLCO9IF_n4q9PbhJ_zs)PJ3d4 zt&n1TM-qPa@aTL*MKGNBC$x_&TOY5f2;#sjU*)WFfJ*J~PA4y*o4gd4yI(Lo7%4dq zOfP6%&)}cd)-3#`*7aHfJ{mV+L4RLL9nD0e0M5~Byd%=s4LLgo%VT!;<$Gj{U8x+oJ)Mo6uDb4Jd|6-`Evyt8%?w{b zVnRG#p>2AChlhu235H8w0C1khibL+=z(Ut;A*A=Q4em5#iQ3}-tzx#0SjS%T;@sJsO}Wh;Qs zF}UkCobJpthG4K{I27>=m``xxeBBUsGSL~e>&oA7+7%BE5Azfh%sYV;2zK6ct5#CQ`{q0lvs-U)o?RBwrAZFMQK!M?5?{tfL@Q_8<|rEwJ3gN6E&oN40v* zxN_bxAbBmoR{}WE5+@lS0Q@A<^f0C`*=C8&3=)czOk<|y8kZ+oRmaxW-u0f|bG(4h z!=oJ#jFc1;i-#C|gwwt>!#fJe?^RbXa{X944-XH01cTAu5Qz_=Q5TLF(_*oTUuiPF zV)h^qCIJY6Fb+)_E-eEgwgCJ(Sj5420oO0KRKPxy6ct6Ylc~f93=Gh+o`CXLO@*QE zXD{IM@MtHDE1G+>FXev-VAnK6Fkuy^0=KSTAKm4&FCHEq_7Ds%7!O6&HbIN%%lNvA zzZp7cbubT&6}(L)g$+; zUuFl?ZA3T`$Y|$yJ%EKW9k_CR{EoLXc&>+shlhs=ybPa*M>{4jZ<^nq-TQKam0zMCq4oOsuk0gc*=2O7p6cX}8X7nLYPc(@+_e4`k zM?HX@O@Gpd%=mzW{6Mg;rl#hO_nrJ3czAetcv#2(2X|XiM6Py&*#H0l07*qoM6N<$ Ef1b6ph#T|mXw78W3p+IpcZUqVy*8)X@mjXqL6?b>nKYgF? zpS2*sTDiG%?#!8e_TGo+w;D>=7~~iL003J>Sza3eKnQ~W9EgSj|9IH-GEZ=F$*02x`&-$?Ht@wXKgPz!#`d6&G)3Wqq3X`f`hpn~p6TaP<2nh5<_YJlE0O*-<#x<0&F~F3E@>6{FxqvUkhI{j-Q|L^_$EmNeEd-&9)W& zS%d?>k!2wGziSZV$G8fkf=5f0)Gj-eZ3bk-tjW=oGfhwi#Nc7R6$>~FM9)Vdrx1!K zKv+T9{i@TEM=NWnm(u)~C_VCjV=HmoSQP~j`6)=@#oB}eF=~JRE|PAyRJ19lqP*;& zk?R`2)7-E#Xd^L5^2Hm_8+&N4*w?mv^)J@DnDRvw_Q1z?RR?63$_o9Bh&}`SiFj}K z0gcXPW2?qTO%kmUnPn})FJpj#tU5Wy=7wAU&^yb;#=Of=Z zT7=z^Ep55=u!ynqT|2Y})v9j#Xhe7lK?j(x8C2H!!MhH*>A&^DQ7V~4DJN}s5P(2! z(eAfQ44-(S@>V3^>&({3L!B`Vh@58FzjO4rTsT2RFZj5f@a{1-A2 z{OB}%Jjr5jiSQXcz0ttm1BocY=Sl(kDPIBuOg8?#SL3x54!Hi0Po|9g`EqNkz=sOg zK)-*3lqpy2^c|pAz@7H7uhT)iXYEG_LXpZ~n%Mq9^Oc|If%<}QC=q*>;wzQEmlqj4 zAATlIW&_0S(s6NbQj{4{o10_k^SQ-hm02x``{n0*f0)u%v1H;tKEI+OBC^dA`~Gi= zBhqTBUK}SZ=R^`tCLn5B`uP47Y;cglM%t^+!lrBhW2?z3Q_yQ#s3`)K0G48+w*I&M zc&R{7-v^-y2PY0fe>#;cl|CI{)-RYYfPmNNmM|Yl4#)>k2C?J4p~gbaJe291w)qG- z!g8?&R{)xLhvPyaem@H7TgKM9@**RZqzdZZZM3wHIOi|G{1#KJ>S!XqX4#)nIdsI? za)hRzIx>)}YmgwM`+NZWeZwK#~M7VE0%; zaDX`|wIr)YkU4~8lht$+o|608snBca=O1hk{@F@Cx$k_`2^#R6^j-eXW(<;PiLC_C zV9?6>wLJZWulH3j_nH_A-M6ev>|ovlCG+|5#Gv+8(Uu0#v-lw`v58pdPzZad`xqv z&4Q;?Z{zXe+%m>SI+_Gbkoll^Xu#VV*}KcykNA3LQ~A{x z<+EUJVok~4j3K&uA}6+y8|UzCYl8sk2M~v&%@rI2*Blco2cbVNXzO3h%P4dO{JHqH zot9zBLC39Rjudj|!XKr-FeVN8aD>lj^Q|-OGOt~RsNWvBu3D4neD zu&ZeNgT!TBz?D;edCBn=@`4Y(A2OO}PP__(J%L~U@lsJrc~<-U9vHq4T5Z@ex}w_` z!-&7*h*1m@O^N_#_Y9ZoZB_47n~*mqn4s;;j1LRdox4my04T(jm!m_5am?AlP^_Qz zg<66jhN))p`$J6oF_0aC{?Mc&A z5?)YI^;M#I!$(A2gE1*J`I_Ci_e1Af6p=Z%{lNTF+lHX*DIP!z3%Ponw_)j1iYzx8V9^^0K_`v`;SC1!v8P-DKa7h69a zM>tSvLhSFqW7l|GhiKfAlM8CRLa-O68cd2`N)V$Ol(wHQ-!3%d_z|1&0{HU9n;!>6 zBwtaq|1qSdY76)zxw^LYUiK$TNdGm8s%pr;)QP|~TenrCnx1zTn#$wn5J{?eU^EtV z{{0woyi7nJf=Qv<_no@4zVkUw@r2|TnXdN|+7GDw9s>WABCEJX!&=2QnejiGa1PCB+iU)&-Bpst`XjPh}X$O6i#But6yx3 z5_acRyBwPl-N&{c2nZND%Aru%E$2Ue`>ucmH$?Q={M%ySgUw6!3V$5JY8E@E-7R8h zp>s~WHSOeh?K=!Pf1`>l|E}<}=t(*z?td9WnCcTe={3hj5RO>}7BR&cFy66A92kw$ zc#CBc`XfTM0rnJ+C-k%?9z*;~oBj6S$CGbm2boc%Uk?N_@DggoG=gnl%t1#-=Zan! zGp*4F_|Z17Jz5m}@&&J^x92qgmKKhL3h)T(ow<|YkIow9-+q2r@*m|k4J<1}hbrk1 z%)VLQby^h%#eI0)2No2P)m`T+x&AvEI2;N1CfM>oI~nCtG$H1bi zHG?cBCw@ad!{3lOTopdKFL>}XPV5}hD{HE*1sYAyt*-<>+(oFO;8XJ*C%xT$8R37= zh|kG{C6Q9RyY&g<7sdN+F;iS{enAI7S{PAmorGf9UC6ax6^JF!^6+XTigLPjPU(@v zYz8?uWRb?(_4xJeQxORQ>7f_pk_TT7&7VIf(Sx(2C`0yuST<-E1Nj^7&^tO%XPYfrrhf zQmBJ5XF*?6X(*O*HE2u$uo{2FGiSwMOH*|U>8p=7zW2U6tTcSU+s5uC{I}GT4F&b@ z%`qX(0`6c*t~CmXI)$3_)YUi-Yb#96$-x0b-@rP2D)iC-z5sR6x5xlRu0G$XVpB&9 zQ-`qwT=ynfsYI_k0}g5Mqd9x)=Pwo1ZiO@4Lp!4PZCLyHqUA68iHP`?8zvGWmFVsO z1idq#$b@RxdRAu&e*P+0ee_7A#MQO~isQz{0OwwpYqnsDttVJ(@1if6M01T_ko|ta9n-wK)CVI+Xxx8|LIA%J z(ZL367?hh_Jr5_bLX$gc^9quFIxq7tkzj54f@7P96o_Lj(CMhlD|4RMjQ9zkesUkO z0V?+QXBG{NjlBev7jaju_2;Bsq1f3qWTr+VS=ipP4fYmw@olv7LvG?9Q2`-R#}9wW zU&{@xU=)H#XEcAjdL%fk4s5z0R2P#E3XK~AKyXZoU-p}Q`>++ zy}Fe8y_ZJ4flQoyyO#oxM&=;;Z1sG^RWdNZd@otDKt+ibe}`(1{<=0}vYSZ?6*Pu2 zWdscU-0BuVLe* ziqvbAFGXxwSUqNB)~gn**V>hGeLDS4{TAIqymOBb^81}={IX?nl{)!TX$O zRpk9H-e}f~mEIA4D}EpBoSGSZ`P~myuuwKqP<&4!Ee7IRtG}Tg!-3y6_MjR81qj`O zz8j-K;!5Kh#vm%HO+vVMB!*p7XCgr%Hr+7s7^8!Eyn>RE$+$s>P}z<%$Ech~6Fu@| zf$5r6SYjvB#76nZYq*TW$BXuwPh0ErOVfB>DHihK!?6~iV@~Q%`_!`q&aIDL5v)^$;LLAITNYmkP84>ZD zV^I-N=zT7%3x9(#U%pYGtn5Othb^qXLR)51mR{ujQAXv?v zlKg#GhB6#4H2&AG`L@y9<}i7B5#j7n3du7U*U@&JpFBR%^1O+og{Jspz&@mI@x0J? zw6Q>IJO}_{EwF@x_!F-dceOdwQ3ygM3ziejonMU$X^bLE3U^Rh#L5gNLuyz^=c{cv zqj$J%r(K^9s0I)!bUKPx5@ftHBNwysroC@{$8hNgQ00sn!ds^fCt>y9{4CnA%;do5 zc%!+;KZ|XVeUi^N))=7>;*1nd^n0_&-^05QEHTUm*UXgpX2U8=-& zf!&Gz1B~_T0=hNp5~Fc*$(c^aDO2a-xAo`tXe=)n8$w*EbBm`n8Ri{>ziRm#7&_5b ze|)^g{}9fa`gimrEyT`juY}$n=oc()x>s#j=P3RvwnG|+p}5d_xq!>($&QeBJur93 zoEyX?{k)3kpmJHIqn-bV0ny>$y=e}T^wm*N82>Z0to4u6c>UW92bP73(vze1G2PH= zxR)1>a*`dyuioh`)zTBIc5)gy(hrOQq0$+M2k!4ncz3z*YIK9%`>LrwYv|3cXd5&S znD+3#&3yZv81%Or0}#|vV%xW%Nq}zRD9aG^nMvOB?iMtO+xh_Y( zq598a8Idq8zNEsC*=Ve!NJki%0hh^)OECLAjyICcU*9h~zY_fEg&$lGRY@z%SU}P8 z?-_wvF`0n{M%$i3`UYb{dVyLF^htMZHhI-@J2rp|ykOg_{XdA?cCacCpTb6&LY?3F zgxdA0xz#ggc=?r=g|R#UHi~s-xVM_DKH*nyH_t|~Tyrlb832F`p{iuXH4{k8n7zgy zN*aH?P|aO8j*;#1g`pMo5cV+MG%y5+RcU**4EvhM#AK|!A8h6aW(2#@mS))vZJ{^Qt}H9xH8mlph)0JfRO;GYF!`dk|v1@jzTUmHe&mC^S;{{-Hh=3 zMuA2cDzMb1<9i}mHHjcbjq{eW z^8+tX@TYfh;PSzC5`!+~K+A$VH@)F1(kXvuH|| z(Zu7^Ull#QyT1S&#}yuFjobGk?|=8xEMWw#Hcu~f`Fd2j>`rxpPPbb|3JSc}a;c%X;Cip$AJ7G>3e>@W%nbf zghSwFArG*pd<*z2DA*}dZt*&ZUEhTqmv+~SQa^FDCuqI3>I2sKITvQno_ti>nMSx6 zvOkl733!W^F?b$9tQ!0eYpF5YNl}Lbdi3YccJpj2vSO_2HgA+yMJ|_a>Wt{Q&Eo{C zo0~X_rFd-}QAyo1>SY$Kv1uAQ$;WaE4gVGdax{Wa2jogzK*0-z4gryK-)cDIpF=p} zkO9C}#`3QtamTah`78NtIl_r@~#ejv#>`h%R_w?|E|xZ?ueMw5JDabH-Xc&p_m>bxjkXKbfM0N>}Yexl^+ z=IdUx)k+!STGr&t4gR|U(&FCRYCI zBl0JEMw)%4S64#GZblse-xD&4Md2s~WUVZynuThf-}ifW5Rm6?sjQS4NwvQvI46Lm zwfxEdM;iiA@aA$)JL9ebv;yPdPZ80*E!+b_0aEK;Sj4=yyhVdt87_6BwsJ~D=lcei9inX$abVqlL9e-4(-azx2u`J-yP-p z@@r`+6Wq`u!a^)p(MvaK6X}vu<>h*fZ`uu7u@TzsTnU zXBX0k58X1*Ru#kBEEJ4x-St0u5~_SXxcqLBfBMR3aef~m3M54AJR`+cAwh-G+vz6K z1F}i&ZghlDi#}pGsn+|(u$}3dm^9x zdr0C+2JqjlwfbV9X>D7#3by|-M3K3E;-!qZ3pjM4v~SB_e!ICvY^ZW65Ng}OFEr*) z4&%~MM604-N3AQnKEC+^J>JRYeNfHsMC_ zVj?U_?8%YGG?|L^!RL&|$D_#IucL?zN-!cH^wPMKT{!y)^Fd()3I+Qt#!8PE@;pG))3^l7A4Q-IgfC6-gXrz!&Ws z-S`!_6+%nANyMXy;g1ta9QsZyRB{yd9#0UKH>E(6^2kkbc#S%eO2^Y$^+V^gpcyuW z=})NpYSZBJAE7d7Up6KH~iR~JRnXE>=kDrmOn zMJdSJ`1R|ixC3c&T(kAo&vTLMNH3(GyE%O+3Z1m9yM6(R@E;n|17f4uJBGZ^14npt zzagyiWOk-uzcs{gv%gj29P9qsl*0$%lYQ*dt zf)9flclhY~05u_7=xO0rZ;#m<7Tr!_yOrr7ua2*gDDOqi@}+M|g_MMmdbkbOCq2Fe zz1I^qB)}Pah?}s%1su0~UKc1clHx{&8-(3i;mweL_eSW0K5*9mNdTwN|5-fhdSzFj znj9l@v~4w^lJt_k`?TVRd#VkG=1nZ*Uzk2*qT9v@mwa6Q&XL9?Jo&b$V>?A_@dRH| z`m)U~j)1P`QJ#Lj0kVyAx``OxH8s{$EJn4O}jTerm5LWoei0`2e-mCD_ zLKQwf4)lQ{r8JJr3rWP4gYCa3;iqvBniPhD)m%NjZjz(j>R9>hPOsay4&!zwF9(bR?~IP!=2T-}kucV@wT20=ldeID(O9w8_W$ zdNSR|%5U8A)TTiK8hoPOH4vR$Vn&VR zYw-96ZgE9)5=a6G%_F_uZ24qN<#7y}oDHiFc0WIS=@2~jL)#XOzk;%E$*B^=gJBfV*1eF+#(Ipw(@L$}}(Ixz(>CenD*KxnSx7|X0-ILD5dSF?c4_7VDh6c zNiZJ5Ube_1oxBt>=Lp$#)!+R1*{J&obee`n3)4C8)4jO38k&&uV%XhO2kwvo;Uq<` z>AH+`b27?PD#OM`>r>)*RQf6Q+kv>h7wj1Xr}{yX7Tv>ArV25wgyY}Z#|ip6ayXJY zxt3&wTqz^@`3hu1dweNYe&-9e)M0-XnDQ#530Coc5nfYeOt#K=2x%CI28Dm+xc|oP zAJ|K&BeEn_pw$z$v1IF0#TjL zH_u6My0--dvGm`1c()Zfi4x}KDV&p%WLlds=h8Fia{aYJvxk{LvcX`FI+%jl(hi4> zv1O#(@SgB5#Te4Y&93cpm7=+{(Vy)+Vl-wmgE)EJD0FqRe{>Ik?_xwV6o1@8NrPFj zQLFo_VZ4H-wMzBa>!4ig0SDUVYuwSTKIFM_4TQdV4OCBj=tV8NU;%CQoGB-{16YzD z54;~N9fk^h`&~(B^RzYdNZtyrJc+Ff$^Mi=6#hdV{^(J|u6#pO)j%GtEs4aJ+XEfj zGhPwFd?pTG&XX~+>FDmT8`nvnpV%SjlnMXDWo74if%+gk62yt{hz>(|qC21iNovtr zlvLZO=i^;#jm$`fp&aV9ph5}@MFIgC%J=@OyTZrDJQ!luQmAJyz*;+cLTiFw2p#4X z2d+Kf$=4fJA}JNiC>vV64`lXM29F3w6q_9&w*DSdVb-*biAQpt#N!L}e7s*nx#ftl zH;9>c;q{U?&y=K?bz*KW*{vXx7(oz$ByZ*^y5N)N&2F!*?ptceI}rxQq12MLd1|Vd z-2GU9?DIY z!iBSZu*BLwE#mYXd*uIn0V3uP54$_ZVoE!teEbH4X7+>wq(e6;TyCs*&uDvCDrEQA zDU{UKpMj;86_nYDTnxM2q-7H7!+kw{6lk zmzS#v#QNw98c~yKefp{AI)?R|3&kUg4mUO*;*`tgJmFn@(7+jsKns1JJH;ccp3Qxt zisLNtoxehwm6+2%nR#GxE~@X(+><=QhFfD5PZ&gBM0NAO0lHH_MGlIRklltEXB#S)hhf7x6Y=oGN|J1t(ZQdJf$rp>LfFBRcQp(Vyw%x|2A-*N#~_U9vI>g)#S zHXpnA8)wNc+ZFdDvroPA)S^_?od&M`%|%^TcH4VoAt3Q1v8$)`5iyOyX#sJxyYN}S zuDU3!1ywO-=w?QyD~R8l9FYEuRTUXEGQ7Q6Hx>3)oCA_sR`>3SgSFou@SCZSY$OBF z!zhh|0C=uljF&ll6k`d+&_P)S ziHbc%Vb$FZ{*mxFgkkSX5+sKV{v6V3ZdQ{CvuDM%gerq@bW+ z!htc^3mDcHAc?!B4dmqE*54?r0NUP0`lErQkt^;+MtIad?Ir`Y2sL)D%*0zaE>elN zYZ;#LqPm>NdkkZ_I*_Gj4OS*=2{C&IHI(ZpW3HhIW$rVlJbdWgv+zP>s@-QFwbYB{ z4!7;oWAdj+o8TF*5A(gs?N>9lK4h;8slC57(xFVdXd;+7P4?w|`sAn~xrSp%PBEL_ zKRvCfbUfBBH`0adfm*r$ctGiiwWmQb*;38HAC}w( zuGcHIHxqb6pB2{p9Hil{jFimJK)>h_8xs%y!Fe(S{Us=y<#0bzOnyiV$1XCj1YN%l?`P+0>8r{3~pl0McXd&nyKmJTx zvx~!hdzg?F)l2V{2YXK=wx;ukiXm#(s&vC6D_wJGw&*7>PhV8*itFIn;KoJBB2ELw z<5+LyB+{8WyFUNbfzD^Icz0ybjmn0=nYtOqnfBIa3 zw7%d2sOB#@AzlhN?$n%`l*5>nwT(e=vfwm{wQN2X^1o#k(z#TNmtfFcBfUIR-wMs!wh zs2+&x4oCfQ@VVxK{E7~Kex-~jcK|=#w##7pgQx`$S7b8`XPq~SB)xjll^Z|fMFX`= z{t$5IHHC2Sf5u_K>wHC)pBu$uIdO@EAT)`mX%1ZFxJdhX zIc9{5g~k50%gr+HEt*i>+(%E?Z2(=6e+$qJVTy{SLYyk4Ct+bzehi%^qxiw?6N>jV zB51nhJ+vlF%hKb7AcJt(si)Z+v=`udezzDH%)B;ihii^}3ZZlh`5de;58Ez?V6r_`e5XBE)bCviHUZe`#KQG{3%s^f+=R_G8?zSubovEvL|YYKA>jV1 zyuC_h@?37FZRCpXPss#6Uspyh0iJ zwXvR7RC?@j6tyVvleA@41Uo0QVc7MjPv88#}yA0sE8sV?+q)C^Se(Q zLn>nkBmphtC!8UT7iFVwgfURv z`IYfnA208#ATw`e{lfBt5?;`6Hz_O*0q1Z?emq6j4kjkag5a<5BHv}iTI4V4w2#P) z<}he~yoB_7jd}Su=7T6c_B}$y@5Ouu@{w9RvfBINRqH5=3oKn(zc}CaF#|k;dkjp5 zWq*~Jz!_mjb8**|jCSyn!Ks4wXPlw?k2L2ylC=z6m@JA)6LEl%Z{PCeX(5KpF)sk{ z^{?9nLve2!NWL*%T9HjX6)q5D5i)S=46g$PtW74IN(Q9SR^|bDyp}zLYMsuo>&~6m z@aJ03>UrWH+v>qMS3{A2m55>)QA|u$1t{(g0)#C9=7A*=cypnJ@tiYKKeMb4W{kv2 z#+1FKAC#-fdLy0xM1(vxP2~1XmlyAS6U*=eQmWm;g2I=$cr!~pr9tK#nuZjo=_~e@%1;r3 zpCg=eaC%gnoK*(ahepApy_k#Hr|LLQWMUi6z&s0pyVPI{`LJaEh^+j*Imx;@V0FJF z4(=Wrv1!aMSvl?54YscT#BW;hmLs)W?)``6X(?|7)_NMlXBqNSA}KW3C;4W>`rEF1 z`FhRY?rsjIU6Ae>WpuomkG-Dx{OuM$iFJ;nUC4$@BU0ICH4#0&FMWveC@Fgyc`??K z5j{Q2vrF~$}WXW)ZcKbJ{WXw$`XAj|HhQzT-A@Z{)VA0$>I|JI1 zJAEbg8NcNmHtR_ks{?tS*tnu>JsFB3V4zU(p)SX`=QlGJbqP zSIpbO1%4*%A>knl`2w!rUsL2$s&2*ns=jMU{btX2)T1(1mg1KjfLD+#geryIqG_m! zPQS+d2=U7f@oMkv-&D91oP?#!iu$;>d|`bZSGVqU!lJ-|f<1P-B*u`&Lz;=0Ha1s< zflL4vcP$vAg`WKJYv1F=bTwAwNPB!u*Y8=o52wNqrEns{UqH^Ka6=(3Oe4ka_uD@u zVSla_H-?JAb9_W4m~+#+f(Ogojmcv%HHEnRfB>(_g#bS)z$i8vR*ti^@OGx*4~o@6 zm%34pxbbr>pA%{ck^pMSXbXw^u4=X0dZR@(q#I^tPStAz{z7aJo0lgz!B*CzjNvjJZ zVqR0NSB+do%=NPRwv{$XwGE*RXb3Qey{PocQVD#tv10B|g&1njczz;UKC}Q>f z=~$W3Q0vd;k(2n9Xs7|8Rx;`JeURRkxT>=u@6o!Ep^m5L-j{Bs5C;cC=RJt8U5O!w zKQ7dd{CBM9l)&mUl494@46A`5v5L_2Og-s$sS8l3X7k_lNNh^a{UYS0(l@`sTUX8~ zlP7|#X&RRu#O{&3TVnCg-FoO2a=inr@|QgSAfr_Lib;vRU9~xq9NEz^3J#5EeUm8x z^5dfdAf%yJ5aO(|8j+C&H*#nyrg5(b_u;&T8)aQ|)tB;zN(6ZmA)N^3qo6N}vRi2u z_RIb|hjO&ePHgBchAf>zx&n?WJ8p7vPSxa^5|}-3suk91gjSQiz}pi$VA z)_<8<_VV58F*}kEEot@!uk=E9=+&=*C%9ao>hkJC1S`=ATv?X=XR|~cf4A^vFn6SNa|Dt5bp&Qj;Q0uv4nYQWL?q zZ)kRKH~0s7?h9^&45#8FYhjw$in3BLuxo%mwp{i&> ze6aMM?SeI|$KtB*Pi}t?ceWCg(JBV$Sa4YU>Bp6~=<)6o7l z?3m#cdt9LYX8_}xd(X(F;J~9A@yqB)eschd*FUm%NId~!`Ox-ItuKw@ks`3M3p3fwfHMPFzh|TyL z`cdLmPVaGn&SY~v@+QC^n0QBF6L0#cMszt;a0ob1fEh@qy3A^0Hahl+5YEyFGnp>} zz>o=SJpq}&)F^O0OM*lvr3O)WShgXZ*TWGmhT~eDZSwWqvePyr{4%HiAeFAO5Y(ym zs)1J*Km8R8+G?yCu>k`AG-s|_KgRM;<->>{BIEfngDK(o(VM|UL^4yeJKw)^l#Hq^ zC)o!i!kSXE{Rr_&45Y=4{u+<`V}V**13H)J2e)Zo#=7Tg5J2X{ijTa%=s(&;{EbVr zO3K5Pv-4R&x8Rr&|IE?`b)|lD>|-u+6|QqbY+HF)eV9bg`Ualjif)K(&F)pz`0Dt4 zkn#LyoQB~ndkOzFIUGy_MpQ41PG!>v_r^Zz`#VvS7RmABz2h z!;An&Qw-e9&ipLN)9(tBpp5-X+EUfv6w!@^#)RuH{~e0KzEK(j$282us)AvsMtnY` zUIxrD!-a8W-15X@Mz))c?*9y;SAAbWWPkt2iI=oMCKHu6N!;SB}^D8S@<%_ZZR@Qagd~E|=?NAE;7U z*)WwYO*#^gIb?V(hlLy9sJp!k%@p&1`Q4@WGZ(E?lg&`h4dxn$Hkm0p?A)eP>I=)a z=AYA8$&U|);(sV(n7Yc!%Txoi;*8-{VE?a@OgM~>&&YBih1*YY!YR5>lRG|qEWFKw z<|(~#e4%tyr4{5cPF(KE>n$lkpN}Z7rT|K0MM3S;xpLo|rq^wtTUc)lAOpC)L~%;} z19}2&9V0M5th15++Py9uiny*I>j7a6>Hf9ttmnP`NdMUrVAiVpgxlke+uUS9jxowN zMOQw?^lG^MoE0$Oy>ri+n%XTSeOsk>jh2DZGfVau#Qe8Ts)X6Z%IQ1~o}DC1m|qBW zQbKh8Y?kctX``mLFCwj{0?GlG(L7<)_hpqi9TYGMcwflc{(|vAaZhDrtPnxnBN_3u zL-h+MM`uMrkxzE0-s=ye6C~V8$~eDwE^b7~!YTYb4G$7a3L>d0VpvDndzEYu?DiuL zx7iM(=Ji*Mj6?!`VF|-Bev4c*T=w(ZLc{JAWkzpLQS$cp=l{ZP6$_rOk>-BeG?}bKY6G9+I(S_$aTX>t;D#`WlEdgM- zcWy04Ny=wDN_F_T-HHo2qAGJ^E>b?5zP%|SEY~U$(@;TWCaUvL{8;FV1|16{Xfl&9 z&_yyq>^wI?yF97gk(-G-M4eh!7WV6!C;MRsH(`YQgzSF~L2S&Od$L1SB_8unfxWmj z)5Hj4(e8NRQQ!Z>oXI%_0+1}Mby#MkFKtPW>q~_PBCd_e6BH;jsj6QmnT1q7FKbb% z-s*by{uv1>(0N|!(IL40Oog?HQMDO@A65`6VJg1VZ#_F;sic&ei^ImoLW7eYVv=YI z0C(hen!*}Z7x!|lrhm)IFU~A<5`K{qrHVnxh(1982UOP*%NDHOkha(3b&kMtv6+vv z->gnBm`plmizigb|U(B zy(?6<=@g(ZCVmUg=9t$jILxZ00fuTHrB%yiJt;7^YWc{HTT&oWM88VS(H@ocSRC|H zqLqMfa!bW*I=~xqqmYPQFs_-0aRY6-r}i98A`)HCxe9CWO~%^-`;Ogh0Zuz3BM*R*jMkh1d|kovVJ+!_jd^yeS^xN!8Xu&`1B?yelu z&cs~RYI`CApwnhuS=;HgJjzrc`&ZGz*)fk5t>c}6;h~Y)+2H~jgKL(QtY>0W!x4p>#D9rnwS4-xQL=@XQZ9ori|RhZ3n4Jn z%VnO=ZL!TvQkaAvuI%8yVGide=sp)I3r1H_>&kn)p2o@d+8RHb*n@pe&RL&accqT7 zx<_aM$vWxjH3%v6bYs-ElOxSa0g=R{=oV=p9X#4@ z#(vwTyXmi2@P%n(HTJF6tC3imw$kKK_a3U!-~I?r6+PXWVn#1}mU-v|*Og)M&SuO| zb&GVKoKPx+rP&(gw4daplwJ2G~f>k28EgN@MkF+9tTV#I?eJk#%azyCqQT zcx-R6UUqh(;phA|(%Bm|S|!$*J8#W6SaQ3emc4o>dXPWqXS?L(MNE<$f@fM~I%>{2yx#9hZalD$Pe6^I2BLY^hdWzT=Z78n@K90Vw z0gCGh^Y@tY^40BfkJEs#b|TmTvey!hVlKVBzp(gCDBRReGVWUgmGv|iSzMq-y08Zj zzJK!@E+dnj;!5xlW?j5F8J4)m(!Ptx+xpaF{Mw5$;pYn~nw+#Bp4ha7wSCdqMA`^6 zcOO^U=8-ME7LlqK)4Zf0_5R+qAFEY;Fm1M?eA_3GKrazKG3Qp`YFOUSmZZ_%g^!PQ zBmU^bRY;7fpTDP$XdAZ8S+9X*&6nLvB`q{{4bt7eVW6HY2+@0F(RQN8l0w2Y`Vdyl zKvMEwh6RPv(-r8uUwq|tzgytJ{RHZal;RX4J7K4obFZh{5#=oiHnT~ikxt+TK*JKV zRLTFjr=K~N3e-?H%mF^kU=y;-JGi)C-qXGC=~KNAMX>Zo&bSj|poAXMv@-vE^X}X|ouj1o#6U7&~#~cWe zMDWV>RJNhZ6Sn+$HBge7gtM3`oI3mVdPMv_O#4oWP(@30&10z2tawk|Ke<|T55NEj zt+6uMFyJYE-$p?9za}JrCcv+-^Mgs9_OuTjkA=J%Ntg!^&{lJNf%FN=AVaVLy(CC@ z9bWJjWTc#*^mB0*(B9;tcTuWzJZw55AA{D81``J;?pyMK+q?TTF~$kjJl;N_Qd_ZRo=mw3FL_3%nJ9CmkLE(%zy~m3WgUUR*st;k6Dwp3{15n#=-Yua zQjK|;LQcjR2E2IhyUZU3PeY?So~Titl3`LaEkW8GX|1HNm@F#gM-mN58#dr=ep0W* zT$v1+e3W?hSlqZcYBPXh0TP#4LKlqNJFq_4(kkMNtGz__N1|=w_ z5ZN@r8fZ@S9a|YCuc}0--Yny=wy987hKpEi}-V>#?tsoqf|~L4W7Gn zN(EC>G=C%X>4G%rz-*!fgc}BAucrpI%`NlZMeBmPfe88pj0ITYa8O+h8dzKPeP75g&&ujs9Ub=rDZ#AQ zJZc9IEKKk_f4eQ7lh@ec&Plcu@$=Z;RC>{9T*oIfWiCs6+Ze@BXgzroxeGA7V`k=B zUiV_@C@4QrRAw2&_b7?5!wuil5R3RsfUaLZKaES;^71o%8D;0UcS>G9yrc7-m6J9y zzbeG0`$t*F^D`fT@p=tV_^(UQj0`)7=TfD#XgAtW{4N;uY;=J#Fi4-Y+91JM_ZYyF zG{?)4i}PVv^%{cBU>rGEdYCmulPm!^ZD3(@rqGpSJn&tS_TILXJ{FrRXx&vWcJ5i5 zx*5(f?4xxxuw7t5K6|65{@haS7Ps|Y3hvFd5zP*G%zzUh17Sfg1_sTFHv;fx%E$*i zMjD6@GJ+KJWU8Zjp;*61Z;1N3=j5S&O6IcfU(*+)z+SlDxq~SX(*Hjv&81q&#-KcC zrhdr+FV=qWnChV|+I?(S~iUNg~HhBzqipNVvQM}zJfpKPD*7(5IIgnYCQzP!Id=Wl?(UQggZ2)UGb zR?W}>;0~8aEnW$nFM+Id1Fq#+c(*{RVWq^e* zrbgr>0XDX?YQ*{rHq2li{DB6s3lCCWjQPFpU9eGnzc*tYA@suQ))J(*U?jJ@wx&f2 zhwemx9cR-{R!X*f&&ig7?YXyv6Lx@7W2$_*s1I@IsYGtXMko-6px&y!ul_$+XnOzv zWZQrL7a*2CwETN~5P#08LQz>mlrR$bqZY+egsU75?gzWW&U#-IZ&FG1y#r<0=Wyxk zqu|iXHvh1;O%rJf>072>JsKLXcaz`>+)o?4h~IScqW&bX3fg7)*kHAFygJGr`O+{) z4A=$u_~wThhQFitR=WUk|H0bY8XzDAhuJi6a<@CT&K>bmZ#bN9AU~1%T~W5cO+0=~ zu{pw71bMo<)Z&6?e5wx(i4_Jp^+wI;{J#Jg1n2t?6lIH@ zj>pY(DEWg2{rSBIec2COb=^y}6D*&FD18wVZh7D7a85*3lV4)szW_H*IMD#%w~vS^AOlmx} z2EbVww`@Ax+CER~=^oHc!q?EScC?}t1)1f1BAms{1NH3k-0F$hWrp^9R~|`GakWJj`&z>=j~wH-qpxSgu1^@tdHEjMGs8fU0h| znn5dJsnDqXFz}v2`Ab8g)3r@c@bK_(EwNbgDG;vFw3x7A^PA=KPZI6k8b1vR{{Vy# zY##P-3~z(^bC?v-d@b!Xu~_O35U7^XZy$UI3u+vhE)tC+xPK@zVkR8W^|0zT1m75FSA;^9@W{+Gj@TtzNy_w|n>K zBm*2zK)Yrc;+#d-162Q$)$I=VC6@eQnD}!%19f*gAhjqka z8-50&-UXIB*?hRQy}ZY3#+{Ah!1*O05dzpM&E7Ww#NeZcF)p;%BAW$6Rg-r!P^WQu zMk?}*>c#V#^!o7TGkWrpW$%OQ1`Q!@OgOj5$Y4kG5#DL0^<`-$8r<`>N`4=`|rGS_93TZ@$m4l z2USg;3qaHK9wMJ!N^ zfh;c3y1uow{mGh|1#L!rc?q9~M;{{=OWp>;18`Qs{#};1`=qWs4-XFy4_D$Pd>$Ts zkWNiZ+{3_^pz$ieB3i-tArSaMBr@Y*0sVS-czAf2z)Sc%Jo+$ob!+^kLrU%d<5K`G zfnC4Tgts8^%RnITU`53YR}UWd@bK{PI2kYD^YG{=G&F3SqA0nKGw5mn7s0$p`YsSX zON@lyKj#k>736yHQV$Og509?#5 tu val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes) test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes) - print(f'{len(train)} | {len(val)} | {len(test)} | {train.X.shape[1]} | {train.n_classes} | {train.n_classes}') - return train, val, test diff --git a/quapy/method/_helper.py b/quapy/method/_helper.py new file mode 100644 index 0000000..0e5792a --- /dev/null +++ b/quapy/method/_helper.py @@ -0,0 +1,107 @@ +""" +Internal helper utilities shared by quantification methods. +""" + +import numpy as np +from sklearn.metrics import confusion_matrix +from sklearn.preprocessing import LabelEncoder + + +def _get_abstention_calibrators(): + try: + from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling + except ImportError as exc: + raise ImportError( + "Posterior calibration for EMQ requires the optional 'abstention' package." + ) from exc + return { + 'nbvs': NoBiasVectorScaling(), + 'bcts': TempScaling(bias_positions='all'), + 'ts': TempScaling(), + 'vs': VectorScaling(), + } + + +def _get_cvxpy(): + try: + import cvxpy as cp + except ImportError as exc: + raise ImportError( + "RLLS requires the optional 'cvxpy' package." + ) from exc + return cp + + +def _labels_to_indices(labels, classes): + encoder = LabelEncoder().fit(classes) + return encoder.transform(labels) + + +def _rlls_check_mode(mode): + valid = {'soft', 'hard'} + if mode not in valid: + raise ValueError(f'unknown mode {mode!r}; valid ones are {valid}') + return mode + + +def _rlls_joint_distribution(posteriors, labels, classes, mode='soft'): + mode = _rlls_check_mode(mode) + posteriors = np.asarray(posteriors, dtype=float) + labels = np.asarray(labels) + n_samples, n_classes = posteriors.shape + assert n_classes == len(classes), 'wrong number of posterior columns' + + if mode == 'hard': + pred = np.argmax(posteriors, axis=1) + encoded_labels = _labels_to_indices(labels, classes) + joint = confusion_matrix(encoded_labels, pred, labels=np.arange(n_classes)).T.astype(float) + return joint / n_samples + + joint = np.zeros((n_classes, n_classes), dtype=float) + for class_index, class_ in enumerate(classes): + idx = labels == class_ + if idx.any(): + joint[:, class_index] = posteriors[idx].sum(axis=0) + return joint / n_samples + + +def _rlls_predicted_marginal(posteriors, mode='soft'): + mode = _rlls_check_mode(mode) + posteriors = np.asarray(posteriors, dtype=float) + if mode == 'soft': + return posteriors.mean(axis=0) + + pred = np.argmax(posteriors, axis=1) + counts = np.bincount(pred, minlength=posteriors.shape[1]).astype(float) + return counts / counts.sum() + + +def _rlls_compute_3deltaC(n_classes, n_train, delta): + return 3 * ( + 2 * np.log(2 * n_classes / delta) / (3 * n_train) + + np.sqrt(2 * np.log(2 * n_classes / delta) / n_train) + ) + + +def _rlls_compute_weights(C_zy, qz, pz, rho, clip=False): + cp = _get_cvxpy() + + n_classes = C_zy.shape[0] + theta = cp.Variable(n_classes) + b = qz - pz + objective = cp.Minimize(cp.pnorm(C_zy @ theta - b) + rho * cp.pnorm(theta)) + constraints = [-1 <= theta] + problem = cp.Problem(objective, constraints) + + try: + problem.solve(verbose=False, solver=cp.SCS) + except cp.error.SolverError: + problem.solve(verbose=False, solver=cp.SCS, use_indirect=True) + + if theta.value is None: + raise RuntimeError('RLLS optimization failed to produce a solution') + + w = 1 + np.asarray(theta.value, dtype=float) + if clip and np.any(w < 0): + w = np.clip(w, 0, None) + return w diff --git a/quapy/tests/test_functional.py b/quapy/tests/test_functional.py new file mode 100644 index 0000000..97e2307 --- /dev/null +++ b/quapy/tests/test_functional.py @@ -0,0 +1,27 @@ +import unittest + +import numpy as np + +import quapy.functional as F + + +class TestFunctional(unittest.TestCase): + + def test_ternary_search_binary(self): + def loss(prev): + return (prev[1] - 0.37) ** 2 + + result = F.argmin_prevalence(loss, n_classes=2, method='ternary_search') + self.assertTrue(np.allclose(result.sum(), 1.0)) + self.assertAlmostEqual(result[1], 0.37, places=3) + + def test_ternary_search_multiclass_not_supported(self): + def loss(prev): + return np.sum((prev - np.array([0.2, 0.3, 0.5])) ** 2) + + with self.assertRaises(AssertionError): + F.argmin_prevalence(loss, n_classes=3, method='ternary_search') + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_plot.py b/quapy/tests/test_plot.py new file mode 100644 index 0000000..3f5a5a5 --- /dev/null +++ b/quapy/tests/test_plot.py @@ -0,0 +1,41 @@ +import unittest + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + HAS_MATPLOTLIB = True +except ImportError: + plt = None + HAS_MATPLOTLIB = False + +import numpy as np + +import quapy as qp + + +@unittest.skipUnless(HAS_MATPLOTLIB and qp.plot is not None, 'matplotlib is not available') +class TestPlot(unittest.TestCase): + + def test_plot_simplex_smoke(self): + rng = np.random.default_rng(0) + true_prev = np.array([0.2, 0.3, 0.5]) + cloud = rng.dirichlet(alpha=30 * true_prev, size=50) + + fig, ax = plt.subplots(figsize=(5, 5)) + fig, ax = qp.plot.plot_simplex( + point_layers=[ + {'points': cloud, 'label': 'cloud', 'style': {'s': 8, 'alpha': 0.2}}, + {'points': true_prev, 'label': 'target', 'style': {'s': 50, 'color': 'black'}}, + ], + region_layers=[ + {'fn': lambda p: p[:, 2] >= 0.4, 'label': 'high class-3', 'color': 'green', 'alpha': 0.2}, + ], + density_function=lambda p: np.exp(-25 * np.sum((p - true_prev) ** 2, axis=1)), + class_names=['A', 'B', 'C'], + ax=ax, + ) + + self.assertIs(fig, ax.figure) + self.assertGreaterEqual(len(ax.collections), 2) + plt.close(fig) From b29966797a387a295f9bb438fb073a215e08fb42 Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Sat, 4 Jul 2026 18:49:44 +0200 Subject: [PATCH 29/43] Fix 10 correctness bugs and clean up warnings/logging in core library Correctness: - OneVsAllAggregative.aggregation_fit: fix undefined variable and call to the nonexistent aggregate_fit (should be aggregation_fit) - solve_adjustment: stop mutating the caller's fitted arrays in place for method='invariant-ratio' - LabelledCollection.join(): fix classes always being None due to ndarray.sort() returning None; now unions each collection's own classes_ so a class absent from a particular join is kept at zero prevalence - Rename the duplicate newSVMKLD (nkld variant) to newSVMNKLD so both loss variants are reachable - EMQ/DyS/DMy: resolve n_jobs via qp._get_njobs() like the other methods, so qp.environ['N_JOBS'] is respected - AggregativeMedianEstimator: drop backend='threading' (global np.random state mutated via temp_seed is not thread-safe); use the safe process based default instead - NeuralClassifier: default device now 'cpu', matching its own docstring - ConfidenceEllipseSimplex: narrow bare except to np.linalg.LinAlgError - SVMperf: stop merging stderr into stdout so failures report the actual subprocess error instead of crashing with AttributeError - ConfidenceRegionABC: replace @lru_cache on bound methods (leaked every instance for the process lifetime) with per-instance caching Style/quality: - Replace print() with warnings.warn()/logging across aggregative.py, base.py, meta.py, model_selection.py, classification/neural.py, method/_neural.py, classification/svmperf.py, data/reader.py, data/datasets.py; also fixes a `raise RuntimeWarning(...)` in EMQ that would have crashed instead of warning - Remove dead duplicate class MedianEstimator2 in meta.py - Rename misleading _compute_tpr(TP, FP) parameter to FN, matching what callers actually pass - Replace argparse.ArgumentError misuse with ValueError - Remove commented-out dead code in protocol.py - _lequa.py: fix CSV-parse failure raising an unrelated UnboundLocalError instead of a clear ValueError Co-Authored-By: Claude Sonnet 5 --- quapy/classification/neural.py | 14 ++++--- quapy/classification/svmperf.py | 13 +++--- quapy/data/_lequa.py | 3 +- quapy/data/base.py | 4 +- quapy/data/datasets.py | 14 ++----- quapy/data/reader.py | 4 +- quapy/functional.py | 6 ++- quapy/method/_neural.py | 8 ++-- quapy/method/_threshold_optim.py | 6 +-- quapy/method/aggregative.py | 70 +++++++++++++++----------------- quapy/method/base.py | 5 ++- quapy/method/confidence.py | 20 +++++---- quapy/method/meta.py | 66 ++---------------------------- quapy/model_selection.py | 3 +- quapy/plot.py | 2 - quapy/protocol.py | 6 --- 16 files changed, 92 insertions(+), 152 deletions(-) diff --git a/quapy/classification/neural.py b/quapy/classification/neural.py index 8d78d7c..4cdd0d6 100644 --- a/quapy/classification/neural.py +++ b/quapy/classification/neural.py @@ -1,3 +1,4 @@ +import logging import os from abc import ABCMeta, abstractmethod from pathlib import Path @@ -42,7 +43,7 @@ class NeuralClassifierTrainer: batch_size=64, batch_size_test=512, padding_length=300, - device='cuda', + device='cpu', checkpointpath='../checkpoint/classifier_net.dat'): super().__init__() @@ -63,7 +64,7 @@ class NeuralClassifierTrainer: self.learner_hyperparams = self.net.get_params() self.checkpointpath = checkpointpath - print(f'[NeuralNetwork running on {device}]') + logging.getLogger(__name__).info(f'NeuralNetwork running on {device}') os.makedirs(Path(checkpointpath).parent, exist_ok=True) def reset_net_params(self, vocab_size, n_classes): @@ -198,14 +199,15 @@ class NeuralClassifierTrainer: if self.early_stop.IMPROVED: torch.save(self.net.state_dict(), checkpoint) elif self.early_stop.STOP: - print(f'training ended by patience exhasted; loading best model parameters in {checkpoint} ' - f'for epoch {self.early_stop.best_epoch}') + logging.getLogger(__name__).info( + f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' + f'for epoch {self.early_stop.best_epoch}') self.net.load_state_dict(torch.load(checkpoint)) break - print('performing one training pass over the validation set...') + logging.getLogger(__name__).info('performing one training pass over the validation set...') self._train_epoch(valid_generator, self.status['tr'], pbar, epoch=0) - print('[done]') + logging.getLogger(__name__).info('done') return self diff --git a/quapy/classification/svmperf.py b/quapy/classification/svmperf.py index 71f2ac3..3fc48f6 100644 --- a/quapy/classification/svmperf.py +++ b/quapy/classification/svmperf.py @@ -1,3 +1,4 @@ +import logging import random import shutil import subprocess @@ -79,14 +80,14 @@ class SVMperf(BaseEstimator, ClassifierMixin): cmd = ' '.join([self.svmperf_learn, self.c_cmd, self.loss_cmd, traindat, self.model]) if self.verbose: - print('[Running]', cmd) - p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT) + logging.getLogger(__name__).info(f'[Running] {cmd}') + p = subprocess.run(cmd.split(), stdout=PIPE, stderr=PIPE) if not exists(self.model): - print(p.stderr.decode('utf-8')) + logging.getLogger(__name__).error(p.stderr.decode('utf-8')) remove(traindat) if self.verbose: - print(p.stdout.decode('utf-8')) + logging.getLogger(__name__).info(p.stdout.decode('utf-8')) return self @@ -125,11 +126,11 @@ class SVMperf(BaseEstimator, ClassifierMixin): cmd = ' '.join([self.svmperf_classify, testdat, self.model, predictions_path]) if self.verbose: - print('[Running]', cmd) + logging.getLogger(__name__).info(f'[Running] {cmd}') p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT) if self.verbose: - print(p.stdout.decode('utf-8')) + logging.getLogger(__name__).info(p.stdout.decode('utf-8')) scores = np.loadtxt(predictions_path) remove(testdat) diff --git a/quapy/data/_lequa.py b/quapy/data/_lequa.py index 7e4cf52..93299d4 100644 --- a/quapy/data/_lequa.py +++ b/quapy/data/_lequa.py @@ -187,8 +187,7 @@ class ResultSubmission: try: df = pd.read_csv(path, index_col=0) except Exception as e: - print(f'the file {path} does not seem to be a valid csv file. ') - print(e) + raise ValueError(f'the file {path} does not seem to be a valid csv file: {e}') return ResultSubmission.check_dataframe_format(df, path=path) @classmethod diff --git a/quapy/data/base.py b/quapy/data/base.py index 9bdf135..934d0f0 100644 --- a/quapy/data/base.py +++ b/quapy/data/base.py @@ -329,7 +329,9 @@ class LabelledCollection: else: raise NotImplementedError('unsupported operation for collection types') labels = np.concatenate([lc.labels for lc in args]) - classes = np.unique(labels).sort() + # union of each collection's own classes_, so a class declared but absent from + # this particular join (e.g. an empty fold) is preserved at zero prevalence + classes = np.unique(np.concatenate([lc.classes_ for lc in args])) return LabelledCollection(instances, labels, classes=classes) @property diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index fd2383a..a1d5d54 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -1,3 +1,4 @@ +import logging import os from contextlib import contextmanager import zipfile @@ -212,8 +213,9 @@ def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_hom if dataset_name in {'semeval13', 'semeval14', 'semeval15'}: trainset_name = 'semeval' testset_name = 'semeval' if for_model_selection else dataset_name - print(f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common " - f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}") + logging.getLogger(__name__).info( + f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common " + f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}") else: if dataset_name == 'semeval' and for_model_selection==False: raise ValueError('dataset "semeval" can only be used for model selection. ' @@ -1152,11 +1154,3 @@ def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home return Dataset(train, test, name=dataset_name) -if __name__ == '__main__': - #train, val, test = _fetch_image_embedding_splits(dataset_name='mnist', embedding='logits') - #print(train) - #print(val) - #print(test) - - dataset = fetch_image_embeddings(dataset_name='svhn', embedding='features', heldout_only=True) - print(dataset) \ No newline at end of file diff --git a/quapy/data/reader.py b/quapy/data/reader.py index 88791e3..5c95b49 100644 --- a/quapy/data/reader.py +++ b/quapy/data/reader.py @@ -1,3 +1,5 @@ +import logging + import numpy as np from scipy.sparse import dok_matrix from tqdm import tqdm @@ -30,7 +32,7 @@ def from_text(path, encoding='utf-8', verbose=1, class2int=True): all_sentences.append(sentence) all_labels.append(label) except ValueError: - print(f'format error in {line}') + logging.getLogger(__name__).warning(f'format error in {line}') return all_sentences, all_labels diff --git a/quapy/functional.py b/quapy/functional.py index ca47458..a1fc578 100644 --- a/quapy/functional.py +++ b/quapy/functional.py @@ -650,7 +650,11 @@ def solve_adjustment( if method == "inversion": pass # We leave A and B unchanged elif method == "invariant-ratio": - # Change the last equation to replace it with the normalization condition + # Change the last equation to replace it with the normalization condition; + # copy first so this does not mutate the caller's arrays (np.asarray above + # returns the same object, not a copy, when the input is already float64) + A = A.copy() + B = B.copy() A[-1, :] = 1.0 B[-1] = 1.0 else: diff --git a/quapy/method/_neural.py b/quapy/method/_neural.py index 404090f..12d61b7 100644 --- a/quapy/method/_neural.py +++ b/quapy/method/_neural.py @@ -1,3 +1,4 @@ +import logging import os from pathlib import Path import random @@ -173,7 +174,7 @@ class QuaNetTrainer(BaseQuantifier): order_by=0 if data.binary else None, **self.quanet_params ).to(self.device) - print(self.quanet) + logging.getLogger(__name__).debug(self.quanet) self.optim = torch.optim.Adam(self.quanet.parameters(), lr=self.lr) early_stop = EarlyStop(self.patience, lower_is_better=True) @@ -188,8 +189,9 @@ class QuaNetTrainer(BaseQuantifier): if early_stop.IMPROVED: torch.save(self.quanet.state_dict(), checkpoint) elif early_stop.STOP: - print(f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' - f'for epoch {early_stop.best_epoch}') + logging.getLogger(__name__).info( + f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' + f'for epoch {early_stop.best_epoch}') self.quanet.load_state_dict(torch.load(checkpoint)) break diff --git a/quapy/method/_threshold_optim.py b/quapy/method/_threshold_optim.py index 628f01a..9d7624a 100644 --- a/quapy/method/_threshold_optim.py +++ b/quapy/method/_threshold_optim.py @@ -110,10 +110,10 @@ class ThresholdOptimization(BinaryAggregativeQuantifier): TN = np.logical_and(y == y_, y == self.neg_label).sum() return TP, FP, FN, TN - def _compute_tpr(self, TP, FP): - if TP + FP == 0: + def _compute_tpr(self, TP, FN): + if TP + FN == 0: return 1 - return TP / (TP + FP) + return TP / (TP + FN) def _compute_fpr(self, FP, TN): if FP + TN == 0: diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index a7283b7..5e57ee4 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -1,5 +1,5 @@ +import warnings from abc import ABC, abstractmethod -from argparse import ArgumentError from copy import deepcopy from typing import Callable, Literal, Union import numpy as np @@ -82,9 +82,9 @@ class AggregativeQuantifier(BaseQuantifier, ABC): (f'when {val_split=} is indicated as an integer, it represents the number of folds in a kFCV ' f'and must thus be >1') if val_split==5 and not fit_classifier: - print(f'Warning: {val_split=} will be ignored when the classifier is already trained ' - f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} ' - f'to None to avoid this warning.') + warnings.warn(f'{val_split=} will be ignored when the classifier is already trained ' + f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} ' + f'to None to avoid this warning.') self.val_split=None if val_split!=5: assert fit_classifier, (f'Parameter {val_split=} has been modified, but {fit_classifier=} ' @@ -343,8 +343,8 @@ class AggregativeSoftQuantifier(AggregativeQuantifier, ABC): """ if not hasattr(self.classifier, self._classifier_method()): if adapt_if_necessary: - print(f'warning: The learner {self.classifier.__class__.__name__} does not seem to be ' - f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).') + warnings.warn(f'The learner {self.classifier.__class__.__name__} does not seem to be ' + f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).') self.classifier = CalibratedClassifierCV(self.classifier, cv=5) else: raise AssertionError(f'error: The learner {self.classifier.__class__.__name__} does not ' @@ -838,7 +838,7 @@ class EMQ(AggregativeSoftQuantifier): self.exact_train_prev = exact_train_prev self.calib = calib self.on_calib_error = on_calib_error - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) @classmethod def EMQ_BCTS(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, on_calib_error="raise", n_jobs=None): @@ -875,15 +875,15 @@ class EMQ(AggregativeSoftQuantifier): def _check_init_parameters(self): if self.val_split is not None: if self.exact_train_prev and self.calib is None: - raise RuntimeWarning(f'The parameter {self.val_split=} was specified for EMQ, while the parameters ' - f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an ' - f'unnecessary overload.') + warnings.warn(f'The parameter {self.val_split=} was specified for EMQ, while the parameters ' + f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an ' + f'unnecessary overload.', RuntimeWarning) else: if self.calib is not None: - print(f'[warning] The parameter {self.calib=} requires the val_split be different from None. ' - f'This parameter will be set to 5. To avoid this warning, set this value to a float value ' - f'indicating the proportion of training data to be used as validation, or to an integer ' - f'indicating the number of folds for kFCV.') + warnings.warn(f'The parameter {self.calib=} requires the val_split be different from None. ' + f'This parameter will be set to 5. To avoid this warning, set this value to a float value ' + f'indicating the proportion of training data to be used as validation, or to an integer ' + f'indicating the number of folds for kFCV.') self.val_split = 5 def classify(self, X): @@ -945,14 +945,14 @@ class EMQ(AggregativeSoftQuantifier): requires_predictions = (self.calib is not None) or (not self.exact_train_prev) if P is None and requires_predictions: # classifier predictions were not generated because val_split=None - raise ArgumentError(self.val_split, self.__class__.__name__ + - ": Classifier predictions for the aggregative fit were not generated because " - "val_split=None. This usually happens when you enable calibrations or heuristics " - "during model selection but left val_split set to its default value (None). " - "Please provide one of the following values for val_split: (i) an integer >1 " - "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. " - "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit " - "validation data") + raise ValueError(self.__class__.__name__ + + ": Classifier predictions for the aggregative fit were not generated because " + "val_split=None. This usually happens when you enable calibrations or heuristics " + "during model selection but left val_split set to its default value (None). " + "Please provide one of the following values for val_split: (i) an integer >1 " + "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. " + "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit " + "validation data") if self.calib is not None: calibrator = _get_abstention_calibrators().get(self.calib, None) @@ -1023,7 +1023,7 @@ class EMQ(AggregativeSoftQuantifier): s += 1 if not converged: - print('[warning] the method has reached the maximum number of iterations; it might have not converged') + warnings.warn('the method has reached the maximum number of iterations; it might have not converged') return qs, ps @@ -1143,7 +1143,7 @@ class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.tol = tol self.divergence = divergence self.n_bins = n_bins - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) def _ternary_search(self, f, left, right, tol): """ @@ -1275,7 +1275,7 @@ class DMy(AggregativeSoftQuantifier): self.divergence = divergence self.cdf = cdf self.search = search - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) @classmethod def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None): @@ -1449,9 +1449,9 @@ def newSVMKLD(svmperf_base=None, C=1): return newELM(svmperf_base, loss='kld', C=C) -def newSVMKLD(svmperf_base=None, C=1): +def newSVMNKLD(svmperf_base=None, C=1): """ - SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence + SVM(NKLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence normalized via the logistic function, as proposed by `Esuli et al. 2015 `_. Equivalent to: @@ -1578,7 +1578,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier): return F.normalize_prevalence(prevalences) def aggregation_fit(self, classif_predictions, labels): - self._parallel(self._delayed_binary_aggregate_fit(c, classif_predictions, labels)) + self._parallel(self._delayed_binary_aggregate_fit, classif_predictions, labels) return self def _delayed_binary_classification(self, c, X): @@ -1590,7 +1590,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier): def _delayed_binary_aggregate_fit(self, c, classif_predictions, labels): # trains the aggregation function of the cth quantifier - return self.dict_binary_quantifiers[c].aggregate_fit(classif_predictions[:, c], labels) + return self.dict_binary_quantifiers[c].aggregation_fit(classif_predictions[:, c], labels == c) class AggregativeMedianEstimator(BinaryQuantifier): @@ -1656,8 +1656,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): ((params, X, y) for params in cls_configs), seed=qp.environ.get('_R_SEED', None), n_jobs=self.n_jobs, - asarray=False, - backend='threading' + asarray=False ) else: model = self.base_quantifier @@ -1669,8 +1668,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_fit_aggregation, itertools.product(models_preds, q_configs), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) else: configs = qp.model_selection.expand_grid(self.param_grid) @@ -1678,8 +1676,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_fit, ((params, X, y) for params in configs), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) return self @@ -1692,8 +1689,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_predict, ((model, instances) for model in self.models), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) return np.median(prev_preds, axis=0) diff --git a/quapy/method/base.py b/quapy/method/base.py index 85a0525..85337ed 100644 --- a/quapy/method/base.py +++ b/quapy/method/base.py @@ -1,3 +1,4 @@ +import warnings from abc import ABCMeta, abstractmethod from copy import deepcopy @@ -84,8 +85,8 @@ class OneVsAllGeneric(OneVsAll, BaseQuantifier): assert isinstance(binary_quantifier, BaseQuantifier), \ f'{binary_quantifier} does not seem to be a Quantifier' if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier): - print('[warning] the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; ' - f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}') + warnings.warn('the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; ' + f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}') self.binary_quantifier = binary_quantifier self.n_jobs = qp._get_njobs(n_jobs) diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index c7ac218..6f499e4 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -16,7 +16,6 @@ from sklearn.utils import resample from abc import ABC, abstractmethod from scipy.special import factorial import copy -from functools import lru_cache from tqdm import tqdm """ @@ -64,7 +63,6 @@ class ConfidenceRegionABC(ABC): """ ... - @lru_cache def simplex_portion(self): """ Computes the fraction of the simplex which is covered by the region. This is not the volume of the region @@ -73,9 +71,10 @@ class ConfidenceRegionABC(ABC): :return: float, the fraction of the simplex covered by the region """ - return self.montecarlo_proportion() + if not hasattr(self, '_simplex_portion_cache'): + self._simplex_portion_cache = self.montecarlo_proportion() + return self._simplex_portion_cache - @lru_cache def montecarlo_proportion(self, n_trials=10_000): """ Estimates, via a Monte Carlo approach, the fraction of the simplex covered by the region. This is carried @@ -84,10 +83,13 @@ class ConfidenceRegionABC(ABC): :return: float in [0,1] """ - with qp.util.temp_seed(0): - uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials) - proportion = np.clip(self.coverage(uniform_simplex), 0., 1.) - return proportion + if not hasattr(self, '_montecarlo_proportion_cache'): + self._montecarlo_proportion_cache = {} + if n_trials not in self._montecarlo_proportion_cache: + with qp.util.temp_seed(0): + uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials) + self._montecarlo_proportion_cache[n_trials] = np.clip(self.coverage(uniform_simplex), 0., 1.) + return self._montecarlo_proportion_cache[n_trials] @property @abstractmethod @@ -446,7 +448,7 @@ class ConfidenceEllipseSimplex(ConfidenceRegionABC): try: self.precision_matrix_ = np.linalg.pinv(self.cov_) - except: + except np.linalg.LinAlgError: self.precision_matrix_ = None self.dim = samples.shape[-1] diff --git a/quapy/method/meta.py b/quapy/method/meta.py index 37749e1..4f869e5 100644 --- a/quapy/method/meta.py +++ b/quapy/method/meta.py @@ -1,4 +1,5 @@ import itertools +import logging from copy import deepcopy from typing import Union, List import numpy as np @@ -26,65 +27,6 @@ else: QuaNet = "QuaNet is not available due to missing torch package" -class MedianEstimator2(BinaryQuantifier): - """ - This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the - estimation returned by differently (hyper)parameterized base quantifiers. - The median of unit-vectors is only guaranteed to be a unit-vector for n=2 dimensions, - i.e., in cases of binary quantification. - - :param base_quantifier: the base, binary quantifier - :param random_state: a seed to be set before fitting any base quantifier (default None) - :param param_grid: the grid or parameters towards which the median will be computed - :param n_jobs: number of parllel workes - """ - def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None): - self.base_quantifier = base_quantifier - self.param_grid = param_grid - self.random_state = random_state - self.n_jobs = qp._get_njobs(n_jobs) - - def get_params(self, deep=True): - return self.base_quantifier.get_params(deep) - - def set_params(self, **params): - self.base_quantifier.set_params(**params) - - def _delayed_fit(self, args): - with qp.util.temp_seed(self.random_state): - params, X, y = args - model = deepcopy(self.base_quantifier) - model.set_params(**params) - model.fit(X, y) - return model - - def fit(self, X, y): - self._check_binary(y, self.__class__.__name__) - - configs = qp.model_selection.expand_grid(self.param_grid) - self.models = qp.util.parallel( - self._delayed_fit, - ((params, X, y) for params in configs), - seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs - ) - return self - - def _delayed_predict(self, args): - model, instances = args - return model.predict(instances) - - def predict(self, X): - prev_preds = qp.util.parallel( - self._delayed_predict, - ((model, X) for model in self.models), - seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs - ) - prev_preds = np.asarray(prev_preds) - return np.median(prev_preds, axis=0) - - class MedianEstimator(BinaryQuantifier): """ This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the @@ -213,7 +155,7 @@ class Ensemble(BaseQuantifier): def _sout(self, msg): if self.verbose: - print('[Ensemble]' + msg) + logging.getLogger(__name__).info('[Ensemble] ' + msg) def fit(self, X, y): @@ -402,7 +344,7 @@ def _select_k(elements, order, k): def _delayed_new_instance(args): base_quantifier, data, val_split, prev, posteriors, keep_samples, verbose, sample_size = args if verbose: - print(f'\tfit-start for prev {F.strprev(prev)}, sample_size={sample_size}') + logging.getLogger(__name__).info(f'fit-start for prev {F.strprev(prev)}, sample_size={sample_size}') model = deepcopy(base_quantifier) if val_split is not None: @@ -422,7 +364,7 @@ def _delayed_new_instance(args): tr_distribution = get_probability_distribution(posteriors[sample_index]) if (posteriors is not None) else None if verbose: - print(f'\t--fit-ended for prev {F.strprev(prev)}') + logging.getLogger(__name__).info(f'fit-ended for prev {F.strprev(prev)}') return (model, tr_prevalence, tr_distribution, sample if keep_samples else None) diff --git a/quapy/model_selection.py b/quapy/model_selection.py index 0937fa8..c6af0c1 100644 --- a/quapy/model_selection.py +++ b/quapy/model_selection.py @@ -1,4 +1,5 @@ import itertools +import logging import signal from copy import deepcopy from enum import Enum @@ -91,7 +92,7 @@ class GridSearchQ(BaseQuantifier): def _sout(self, msg): if self.verbose: - print(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}') + logging.getLogger(__name__).info(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}') def __check_error_measure(self, error): if error in qp.error.QUANTIFICATION_ERROR: diff --git a/quapy/plot.py b/quapy/plot.py index e4d2529..9d76aa9 100644 --- a/quapy/plot.py +++ b/quapy/plot.py @@ -483,7 +483,6 @@ def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs best_bucket_methods.append(method_order[method_index]) best_methods.append(best_bucket_methods) salient_methods.update(best_bucket_methods) - print(best_bucket_methods) if binning=='isomerous': fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods))) @@ -827,7 +826,6 @@ def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None): pred_y = posteriors>=0.5 bins = np.linspace(0, 1, nbins + 1) binned_values = np.digitize(posteriors, bins, right=False) - print(np.unique(binned_values)) correct = pred_y == y bin_centers = (bins[:-1] + bins[1:]) / 2 bins_names = np.arange(nbins) diff --git a/quapy/protocol.py b/quapy/protocol.py index 45efa6f..95b8029 100644 --- a/quapy/protocol.py +++ b/quapy/protocol.py @@ -445,11 +445,6 @@ class DirichletProtocol(OnLabelledCollectionProtocol): def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0, return_type='sample_prev'): - #assert ((isinstance(alpha, str) and alpha == 'uniform') or - # isinstance(alpha, Number) or - # (isinstance(alpha, Iterable) and all(isinstance(v, Number) for v in alpha))), \ - # f'wrong type for {alpha=}; expected "uniform", a real scalar, or an array-like of real values' - n_classes = data.n_classes if isinstance(alpha, str) and alpha == 'uniform': self.alpha = np.ones(n_classes, dtype=float) @@ -464,7 +459,6 @@ class DirichletProtocol(OnLabelledCollectionProtocol): super(DirichletProtocol, self).__init__(random_state) self.data = data - #self.alpha = alpha self.sample_size = qp._get_sample_size(sample_size) self.repeats = repeats self.random_state = random_state From ed02be2c8d3473ead8d5794e3cd27782fe4290aa Mon Sep 17 00:00:00 2001 From: Alejandro Moreo Date: Sat, 4 Jul 2026 19:05:06 +0200 Subject: [PATCH 30/43] Add smoke tests for previously-untested modules, fix wasteful svmperf tmpdir creation - Add smoke tests covering data/reader.py, method/_threshold_optim.py, classification/calibration.py, method/confidence.py, and the pure-numpy helpers in method/_bayesian.py (skipping the jax/stan-dependent model code itself, consistent with how the aggregative-method registry already treats it as optional) - SVMperf: stop creating a tempfile.TemporaryDirectory() just to discard it immediately for its .name; generate the path directly instead of doing a pointless create/delete/recreate cycle (cleanup already happens via the class's own __del__) Co-Authored-By: Claude Sonnet 5 --- quapy/classification/svmperf.py | 3 +- quapy/tests/test_bayesian_utils.py | 69 +++++++++++++++++++++++++++++ quapy/tests/test_calibration.py | 35 +++++++++++++++ quapy/tests/test_confidence.py | 62 ++++++++++++++++++++++++++ quapy/tests/test_reader.py | 56 +++++++++++++++++++++++ quapy/tests/test_threshold_optim.py | 33 ++++++++++++++ 6 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 quapy/tests/test_bayesian_utils.py create mode 100644 quapy/tests/test_calibration.py create mode 100644 quapy/tests/test_confidence.py create mode 100644 quapy/tests/test_reader.py create mode 100644 quapy/tests/test_threshold_optim.py diff --git a/quapy/classification/svmperf.py b/quapy/classification/svmperf.py index 3fc48f6..b7ba007 100644 --- a/quapy/classification/svmperf.py +++ b/quapy/classification/svmperf.py @@ -67,8 +67,7 @@ class SVMperf(BaseEstimator, ClassifierMixin): # this would allow to run parallel instances of predict random_code = 'svmperfprocess'+'-'.join(str(local_random.randint(0, 1000000)) for _ in range(5)) if self.host_folder is None: - # tmp dir are removed after the fit terminates in multiprocessing... - self.tmpdir = tempfile.TemporaryDirectory(suffix=random_code).name + self.tmpdir = join(tempfile.gettempdir(), random_code) else: self.tmpdir = join(self.host_folder, '.' + random_code) makedirs(self.tmpdir, exist_ok=True) diff --git a/quapy/tests/test_bayesian_utils.py b/quapy/tests/test_bayesian_utils.py new file mode 100644 index 0000000..8ac6058 --- /dev/null +++ b/quapy/tests/test_bayesian_utils.py @@ -0,0 +1,69 @@ +import unittest + +import numpy as np + +from quapy.method._bayesian import ( + _validate_temperature, + _resolve_dirichlet_prior, + kl_div, + js_div, + normalized, + lambda_inverse, + lambda_forward, +) + + +class TestBayesianUtils(unittest.TestCase): + """ + Smoke tests for the pure-numpy helper functions in method/_bayesian.py. These do not require the + optional jax/stan/numpyro dependencies (unlike BayesianKDEy/BayesianMAPLS themselves), so they can + run regardless of whether `quapy[bayes]` is installed. + """ + + def test_validate_temperature(self): + self.assertEqual(_validate_temperature(2), 2.0) + self.assertEqual(_validate_temperature(0.5), 0.5) + with self.assertRaises(ValueError): + _validate_temperature(0) + with self.assertRaises(ValueError): + _validate_temperature(-1) + with self.assertRaises(ValueError): + _validate_temperature('not-a-number') + + def test_resolve_dirichlet_prior(self): + np.testing.assert_array_equal(_resolve_dirichlet_prior('uniform', n_classes=3), np.ones(3)) + np.testing.assert_array_equal(_resolve_dirichlet_prior(2.5, n_classes=3), np.full(3, 2.5)) + np.testing.assert_array_equal(_resolve_dirichlet_prior([1, 2, 3], n_classes=3), np.array([1., 2., 3.])) + with self.assertRaises(ValueError): + _resolve_dirichlet_prior([1, 2], n_classes=3) # wrong shape + with self.assertRaises(ValueError): + _resolve_dirichlet_prior('unknown-prior', n_classes=3) + + def test_kl_div_and_js_div(self): + p = np.array([0.5, 0.5]) + self.assertAlmostEqual(kl_div(p, p), 0.0, places=6) + self.assertAlmostEqual(js_div(p, p), 0.0, places=6) + + q = np.array([0.9, 0.1]) + self.assertGreater(kl_div(p, q), 0.0) + self.assertGreater(js_div(p, q), 0.0) + # JS divergence is symmetric, unlike KL + self.assertAlmostEqual(js_div(p, q), js_div(q, p), places=6) + + def test_normalized(self): + # normalized() operates on a batch of row vectors, shape (n_samples, n_features) + a = np.array([[3.0, 4.0], [0.0, 0.0]]) + normed = normalized(a) + self.assertAlmostEqual(np.linalg.norm(normed[0]), 1.0, places=6) + # an all-zero row should not raise (guarded division), and stays all-zero + np.testing.assert_array_equal(normed[1], np.array([0.0, 0.0])) + + def test_lambda_inverse_forward_roundtrip(self): + dpq, lam = 0.3, 0.4 + gamma = lambda_inverse(dpq, lam) + recovered_lam = lambda_forward(dpq, gamma) + self.assertAlmostEqual(recovered_lam, lam, places=6) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_calibration.py b/quapy/tests/test_calibration.py new file mode 100644 index 0000000..4bdaae4 --- /dev/null +++ b/quapy/tests/test_calibration.py @@ -0,0 +1,35 @@ +import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + +from quapy.classification.calibration import NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration +from quapy.tests._synthetic import make_labelled_collection + + +class TestCalibration(unittest.TestCase): + + data = make_labelled_collection(n_samples=200, n_features=10, n_classes=3, random_state=23) + + def test_calibration_methods_fit_predict_proba(self): + X, y = self.data.Xy + for calib_cls in [NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration]: + model = calib_cls(LogisticRegression(max_iter=2000), val_split=5) + model.fit(X, y) + posteriors = model.predict_proba(X) + self.assertEqual(posteriors.shape, (len(y), self.data.n_classes)) + np.testing.assert_allclose(posteriors.sum(axis=1), 1.0, rtol=1e-5, + err_msg=f'{calib_cls.__name__} posteriors do not sum to 1') + predictions = model.predict(X) + self.assertEqual(len(predictions), len(y)) + + def test_calibration_with_float_val_split(self): + X, y = self.data.Xy + model = BCTSCalibration(LogisticRegression(max_iter=2000), val_split=0.3) + model.fit(X, y) + posteriors = model.predict_proba(X) + self.assertEqual(posteriors.shape, (len(y), self.data.n_classes)) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_confidence.py b/quapy/tests/test_confidence.py new file mode 100644 index 0000000..c8b9d47 --- /dev/null +++ b/quapy/tests/test_confidence.py @@ -0,0 +1,62 @@ +import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + +from quapy.method.aggregative import PACC +from quapy.method.confidence import ConfidenceIntervals, ConfidenceEllipseSimplex, AggregativeBootstrap +from quapy.tests._synthetic import make_dataset + + +def _dirichlet_samples(n_classes=3, n_samples=300, random_state=0): + rng = np.random.RandomState(random_state) + return rng.dirichlet(np.ones(n_classes) * 5, size=n_samples) + + +class TestConfidenceRegions(unittest.TestCase): + + def test_confidence_intervals_contain_own_mean(self): + samples = _dirichlet_samples() + region = ConfidenceIntervals(samples) + point_estimate = region.point_estimate() + self.assertEqual(region.coverage(point_estimate), 1.) + self.assertEqual(region.n_dim, 3) + + def test_confidence_ellipse_simplex_contains_own_mean(self): + samples = _dirichlet_samples() + region = ConfidenceEllipseSimplex(samples) + point_estimate = region.point_estimate() + self.assertIn(point_estimate, region) + + def test_simplex_portion_is_cached_and_consistent(self): + # regression test for the @lru_cache-on-bound-method memory leak fix: + # results must still be memoized per instance, and two instances must not share state + region1 = ConfidenceEllipseSimplex(_dirichlet_samples(random_state=1)) + region2 = ConfidenceEllipseSimplex(_dirichlet_samples(random_state=2)) + + p1_first = region1.simplex_portion() + p1_second = region1.simplex_portion() + self.assertEqual(p1_first, p1_second) + + p2 = region2.simplex_portion() + self.assertTrue(hasattr(region1, '_simplex_portion_cache')) + self.assertTrue(hasattr(region2, '_simplex_portion_cache')) + # each instance keeps its own cached value + self.assertEqual(region1._simplex_portion_cache, p1_first) + self.assertEqual(region2._simplex_portion_cache, p2) + + def test_aggregative_bootstrap_end_to_end(self): + dataset = make_dataset(n_train=150, n_test=50, n_classes=3, n_features=12, random_state=5) + learner = LogisticRegression(max_iter=2000) + learner.fit(*dataset.training.Xy) + quantifier = AggregativeBootstrap( + PACC(learner, fit_classifier=False), n_test_samples=50, confidence_level=0.9 + ) + quantifier.fit(*dataset.training.Xy) + point_estimate, region = quantifier.predict_conf(dataset.test.X) + self.assertEqual(len(point_estimate), 3) + self.assertEqual(region.coverage(point_estimate), 1.) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_reader.py b/quapy/tests/test_reader.py new file mode 100644 index 0000000..a7ee21c --- /dev/null +++ b/quapy/tests/test_reader.py @@ -0,0 +1,56 @@ +import os +import tempfile +import unittest + +import numpy as np + +from quapy.data.reader import from_text, from_sparse, from_csv, reindex_labels, binarize + + +class TestReader(unittest.TestCase): + + def _write_tmp(self, content, suffix='.txt'): + fd, path = tempfile.mkstemp(suffix=suffix) + with os.fdopen(fd, 'w') as f: + f.write(content) + self.addCleanup(os.remove, path) + return path + + def test_from_text(self): + path = self._write_tmp('1\tthis is positive\n0\tthis is negative\n') + sentences, labels = from_text(path, verbose=0) + self.assertEqual(sentences, ['this is positive', 'this is negative']) + self.assertEqual(labels, [1, 0]) + + def test_from_text_skips_malformed_lines(self): + # a line without a tab separator should be skipped (and warned about), not raise + path = self._write_tmp('1\tgood line\nthis line has no label\n0\tanother good line\n') + sentences, labels = from_text(path, verbose=0) + self.assertEqual(sentences, ['good line', 'another good line']) + self.assertEqual(labels, [1, 0]) + + def test_from_sparse(self): + # format: