diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index 0b7a18c..a384d39 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -1,3 +1,38 @@ +Change Log 0.2.1 +----------------- + +- Improved documentation of confidence regions. Added QuaPy logo :') + +- Added Bayesian KDEy and Bayesian MAPLS quantifiers. + +- Added temperature calibration utilities for Bayesian confidence-aware methods. + +- Added compositional CLR and ILR transformations. + +- Extended KDEy with Aitchison/ILR kernels, shrinkage, and improved numerical stability. + +- Added image-embedding-based datasets including CIFAR10, CIFAR100, CIFAR100coarse, VSHN, FashionMNIST, MNIST. + +- Added TemperatureScalingFromLogits for calibrating pretrained logits. + +- Added DirichletProtocol for prevalence sampling from Dirichlet priors. + +- Added ReadMe method by Daniel Hopkins and Gary King. + +- Internal index in LabelledCollection is now "lazy", and is only constructed if required. + +- Improved unit testing and separated integration tests. + +- Added RLLS (Regularized Learning for Domain Adaptation under Label Shifts) method. + +- Added visualization tools for 3-class problems in the simplex, see also the new example no.19 + +- Deep code revision and improved codebase + +- Added EDx/EDy from quantificationlib (thanks to Pablo and Juanjo!) + + + Change Log 0.2.0 ----------------- @@ -14,6 +49,7 @@ Change Log 0.2.0 in which case the_data is to be used for validation purposes. However, the val_split could be set as a fraction indicating only part of the_data must be used for validation, and the rest wasted... it was certainly confusing. - This change imposes a versioning constrain with qunfold, which now must be >= 0.1.6 + - EMQ has been modified, so that the representation function "classify" now only provides posterior probabilities and, if required, these are recalibrated (e.g., by "bcts") during the aggregation function. - A new parameter "on_calib_error" is passed to the constructor, which informs of the policy to follow @@ -21,13 +57,16 @@ Change Log 0.2.0 - 'raise': raises a RuntimeException (default) - 'backup': reruns by silently avoiding calibration - Parameter "recalib" has been renamed "calib" + - Added aggregative bootstrap for deriving confidence regions (confidence intervals, ellipses in the simplex, or ellipses in the CLR space). This method is efficient as it leverages the two-phases of the aggregative quantifiers. This method applies resampling only to the aggregation phase, thus avoiding to train many quantifiers, or classify multiple times the instances of a sample. See: - quapy/method/confidence.py (new) - the new example no. 16.confidence_regions.py + - BayesianCC moved to confidence.py, where methods having to do with confidence intervals belong. + - Improved documentation of qp.plot module. @@ -129,6 +168,7 @@ Change Log 0.1.8 - New API documentation template. + Change Log 0.1.7 ---------------- @@ -182,7 +222,7 @@ Change Log 0.1.7 - hyperparameters yielding to inconsistent runs raise a ValueError exception, while hyperparameter combinations yielding to internal errors of surrogate functions are reported and skipped, without stopping the grid search. -- DistributionMatching methods added. This is a general framework for distribution matching methods that catters for +- DistributionMatching methods added. This is a general framework for distribution matching methods that caters for multiclass quantification. That is to say, one could get a multiclass variant of the (originally binary) HDy method aligned with the Firat's formulation. @@ -207,4 +247,3 @@ Change Log 0.1.7 any instance of BaseQuantifier), and a subclass of it called OneVsAllAggregative which implements the classify / aggregate interface. Both are instances of OneVsAll. There is a method getOneVsAll that returns the best instance based on the type of quantifier. - diff --git a/README.md b/README.md index 730a433..4110376 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # QuaPy +## version 0.2.1 + QuaPy is an open source framework for quantification (a.k.a. supervised prevalence estimation, or learning to quantify) written in Python. @@ -13,7 +15,7 @@ for facilitating the analysis and interpretation of the experimental results. ### Last updates: -* Version 0.2.0 is released! major changes can be consulted [here](CHANGE_LOG.txt). +* Version 0.2.1 is released! major changes can be consulted [here](CHANGE_LOG.txt). * The developer API documentation is available [here](https://hlt-isti.github.io/QuaPy/index.html) * Manuals are available [here](https://hlt-isti.github.io/QuaPy/manuals.html) @@ -74,6 +76,7 @@ See the [documentation](https://hlt-isti.github.io/QuaPy/manuals.html) for detai * Implementation of many popular quantification methods (Classify-&-Count and its variants, Expectation Maximization, quantification methods based on structured output learning, HDy, QuaNet, quantification ensembles, among others). +* Support for uncertainty quantification via bootstrap-based and Bayesian methods, including confidence intervals and simplex-aware confidence regions. * Versatile functionality for performing evaluation based on sampling generation protocols (e.g., APP, NPP, etc.). * Implementation of most commonly used evaluation metrics (e.g., AE, RAE, NAE, NRAE, SE, KLD, NKLD, etc.). * Datasets frequently used in quantification (textual and numeric), including: diff --git a/TODO.txt b/TODO.txt index 4b80a34..7a36ae1 100644 --- a/TODO.txt +++ b/TODO.txt @@ -16,7 +16,6 @@ scale each value by per-class thresholds, i.e., [0.33*0.1, 0.33*1, 0.33*1]/sum." - [TODO] document confidence in manuals - [TODO] Test the return_type="index" in protocols and finish the "distributing_samples.py" example -- [TODO] Add EDy (an implementation is available at quantificationlib) - [TODO] add ensemble methods SC-MQ, MC-SQ, MC-MQ - [TODO] add HistNetQ - [TODO] add CDE-iteration and Bayes-CDE methods diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html index 9951ba2..c9bfcd6 100644 --- a/docs/build/html/_modules/index.html +++ b/docs/build/html/_modules/index.html @@ -1,78 +1,374 @@ + - - - - - Overview: module code — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + Overview: module code — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/classification/calibration.html b/docs/build/html/_modules/quapy/classification/calibration.html index 6576c6c..ebf5d07 100644 --- a/docs/build/html/_modules/quapy/classification/calibration.html +++ b/docs/build/html/_modules/quapy/classification/calibration.html @@ -1,82 +1,382 @@ + - - - - - quapy.classification.calibration — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.classification.calibration — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ +

Source code for quapy.classification.calibration

+from copy import deepcopy
+
+from sklearn.base import BaseEstimator, clone
+from sklearn.model_selection import cross_val_predict, train_test_split
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils.validation import check_X_y
+import numpy as np
 
 
 # Wrappers of calibration defined by Alexandari et al. in paper <http://proceedings.mlr.press/v119/alexandari20a.html>
@@ -84,7 +384,20 @@
 # see https://github.com/kundajelab/abstention
 
 
-
[docs]class RecalibratedProbabilisticClassifier:
+def _require_abstention_calibration():
+    try:
+        from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
+    except ImportError as exc:
+        raise ImportError(
+            "Calibration methods in quapy.classification.calibration require the optional "
+            "'abstention' package."
+        ) from exc
+    return NoBiasVectorScaling, TempScaling, VectorScaling
+
+
+

+[docs]
+class RecalibratedProbabilisticClassifier:
     """
     Abstract class for (re)calibration method from `abstention.calibration`, as defined in
     `Alexandari, A., Kundaje, A., & Shrikumar, A. (2020, November). Maximum likelihood with bias-corrected calibration
@@ -94,7 +407,10 @@
     pass

 
 
-
[docs]class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
+
+

+[docs]
+class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
     """
     Applies a (re)calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_.
@@ -110,14 +426,16 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
         self.classifier = classifier
         self.calibrator = calibrator
         self.val_split = val_split
         self.n_jobs = n_jobs
         self.verbose = verbose
 
-
[docs]    def fit(self, X, y):
+

+[docs]
+    def fit(self, X, y):
         """
         Fits the calibration for the probabilistic classifier.
 
@@ -135,7 +453,10 @@
                 raise ValueError('wrong value for val_split: the proportion of validation documents must be in (0,1)')
             return self.fit_tr_val(X, y)

 
-
[docs]    def fit_cv(self, X, y):
+
+

+[docs]
+    def fit_cv(self, X, y):
         """
         Fits the calibration in a cross-validation manner, i.e., it generates posterior probabilities for all
         training instances via cross-validation, and then retrains the classifier on all training instances.
@@ -153,7 +474,10 @@
         self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[y], posterior_supplied=True)
         return self

 
-
[docs]    def fit_tr_val(self, X, y):
+
+

+[docs]
+    def fit_tr_val(self, X, y):
         """
         Fits the calibration in a train/val-split manner, i.e.t, it partitions the training instances into a
         training and a validation set, and then uses the training samples to learn classifier which is then used
@@ -171,7 +495,10 @@
         self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[yva], posterior_supplied=True)
         return self

 
-
[docs]    def predict(self, X):
+
+

+[docs]
+    def predict(self, X):
         """
         Predicts class labels for the data instances in `X`
 
@@ -180,7 +507,10 @@
         """
         return self.classifier.predict(X)

 
-
[docs]    def predict_proba(self, X):
+
+

+[docs]
+    def predict_proba(self, X):
         """
         Generates posterior probabilities for the data instances in `X`
 
@@ -190,8 +520,9 @@
         posteriors = self.classifier.predict_proba(X)
         return self.calibration_function(posteriors)

 
+
     @property
-    def classes_(self):
+    def classes_(self):
         """
         Returns the classes on which the classifier has been trained on
 
@@ -200,7 +531,10 @@
         return self.classifier.classes_

 
 
-
[docs]class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the No-Bias Vector Scaling (NBVS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -214,7 +548,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        NoBiasVectorScaling, _, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = NoBiasVectorScaling(verbose=verbose)
         self.val_split = val_split
@@ -222,7 +557,10 @@
         self.verbose = verbose

 
 
-
[docs]class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Bias-Corrected Temperature Scaling (BCTS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -236,7 +574,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, TempScaling, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = TempScaling(verbose=verbose, bias_positions='all')
         self.val_split = val_split
@@ -244,7 +583,10 @@
         self.verbose = verbose

 
 
-
[docs]class TSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class TSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Temperature Scaling (TS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -258,7 +600,8 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, TempScaling, _ = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = TempScaling(verbose=verbose)
         self.val_split = val_split
@@ -266,7 +609,10 @@
         self.verbose = verbose

 
 
-
[docs]class VSCalibration(RecalibratedProbabilisticClassifierBase):
+
+

+[docs]
+class VSCalibration(RecalibratedProbabilisticClassifierBase):
     """
     Applies the Vector Scaling (VS) calibration method from `abstention.calibration`, as defined in
     `Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -280,40 +626,172 @@
     :param verbose: whether or not to display information in the standard output
     """
 
-    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+    def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+        _, _, VectorScaling = _require_abstention_calibration()
         self.classifier = classifier
         self.calibrator = VectorScaling(verbose=verbose)
         self.val_split = val_split
         self.n_jobs = n_jobs
         self.verbose = verbose

 
+
+
+

+[docs]
+class TemperatureScalingFromLogits(BaseEstimator):
+    """
+    Calibrates a matrix of logits by learning a temperature-scaling mapping
+    with the calibration methods from `abstention.calibration`.
+
+    This estimator is useful when the inputs are already logits produced by a
+    pretrained classifier, and the goal is to transform them directly into
+    calibrated posterior probabilities without retraining the underlying model.
+
+    :param bias_corrected: if True, uses Bias-Corrected Temperature Scaling
+        (BCTS); otherwise, uses standard Temperature Scaling (TS)
+    :param verbose: whether the underlying calibrator should display progress
+        information
+    """
+
+    def __init__(self, bias_corrected=False, verbose=False):
+        self.bias_corrected = bias_corrected
+        self.verbose = verbose
+
+

+[docs]
+    def fit(self, X, y):
+        """
+        Fits the logits calibrator.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :param y: array-like of shape `(n_samples,)` containing class labels
+        :return: self
+        """
+        X, y = check_X_y(X, y)
+
+        self.label_encoder_ = LabelEncoder()
+        y_enc = self.label_encoder_.fit_transform(y)
+        self.classes_ = self.label_encoder_.classes_
+
+        n_classes = len(self.classes_)
+        logits_dim = X.shape[1]
+        if n_classes != logits_dim:
+            raise ValueError(
+                f'mismatch between the number of classes ({n_classes}) and the '
+                f'dimensionality of the logits ({logits_dim})'
+            )
+
+        _, TempScaling, _ = _require_abstention_calibration()
+        calibrator = TempScaling(
+            verbose=self.verbose,
+            bias_positions='all' if self.bias_corrected else [],
+        )
+        self.calibrator_ = calibrator
+        self.calibration_function_ = calibrator(X, np.eye(n_classes)[y_enc])
+        return self

+
+
+

+[docs]
+    def predict_proba(self, X):
+        """
+        Converts logits into calibrated posterior probabilities.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :return: array-like of shape `(n_samples, n_classes)` with calibrated
+            posterior probabilities
+        """
+        return self.calibration_function_(X)

+
+
+

+[docs]
+    def predict(self, X):
+        """
+        Predicts class labels after calibration.
+
+        :param X: array-like of shape `(n_samples, n_classes)` containing
+            logits
+        :return: array-like of shape `(n_samples,)` with class label
+            predictions
+        """
+        posteriors = self.predict_proba(X)
+        return self.label_encoder_.inverse_transform(np.argmax(posteriors, axis=1))

+

+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/classification/methods.html b/docs/build/html/_modules/quapy/classification/methods.html index 883f802..7a876f0 100644 --- a/docs/build/html/_modules/quapy/classification/methods.html +++ b/docs/build/html/_modules/quapy/classification/methods.html @@ -1,83 +1,384 @@ + - - - - - quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.classification.methods

-from sklearn.base import BaseEstimator
-from sklearn.decomposition import TruncatedSVD
-from sklearn.linear_model import LogisticRegression
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.decomposition import TruncatedSVD
+from sklearn.linear_model import LogisticRegression
 
 
 

 [docs]
-class LowRankLogisticRegression(BaseEstimator):
+class LowRankLogisticRegression(BaseEstimator):
     """
     An example of a classification method (i.e., an object that implements `fit`, `predict`, and `predict_proba`)
     that also generates embedded inputs (i.e., that implements `transform`), as those required for
@@ -91,13 +392,13 @@
         `Logistic Regression <https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html>`__ classifier
     """
 
-    def __init__(self, n_components=100, **kwargs):
+    def __init__(self, n_components=100, **kwargs):
         self.n_components = n_components
         self.classifier = LogisticRegression(**kwargs)
 
 

 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator.
 
@@ -110,7 +411,7 @@
 
 

 [docs]
-    def set_params(self, **params):
+    def set_params(self, **params):
         """
         Set the parameters of this estimator.
 
@@ -127,7 +428,7 @@
 
 

 [docs]
-    def fit(self, X, y):
+    def fit(self, X, y):
         """
         Fit the model according to the given training data. The fit consists of
         fitting `TruncatedSVD` and then `LogisticRegression` on the low-rank representation.
@@ -148,7 +449,7 @@
 
 

 [docs]
-    def predict(self, X):
+    def predict(self, X):
         """
         Predicts labels for the instances `X` embedded into the low-rank space.
 
@@ -162,7 +463,7 @@
 
 

 [docs]
-    def predict_proba(self, X):
+    def predict_proba(self, X):
         """
         Predicts posterior probabilities for the instances `X` embedded into the low-rank space.
 
@@ -175,7 +476,7 @@
 
 

 [docs]
-    def transform(self, X):
+    def transform(self, X):
         """
         Returns the low-rank approximation of `X` with `n_components` dimensions, or `X` unaltered if
         `n_components` >= `X.shape[1]`.
@@ -188,33 +489,109 @@
         return self.pca.transform(X)

 

 
+
+
+

+[docs]
+class MockClassifierFromPosteriors(BaseEstimator):
+    """
+    Mock classifier that bypasses classifier training when the input instances
+    are already posterior probabilities produced by a pretrained probabilistic
+    classifier.
+
+    :param X: arrays of shape `(n_samples, n_classes)` are interpreted as posterior probabilities
+    """
+
+

+[docs]
+    def fit(self, X, y):
+        self.classes_ = np.sort(np.unique(y))
+        return self

+
+
+

+[docs]
+    def predict(self, X):
+        return np.argmax(X, axis=1)

+
+
+

+[docs]
+    def predict_proba(self, X):
+        return X

+

+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/classification/neural.html b/docs/build/html/_modules/quapy/classification/neural.html index fd5d9b1..42d1b4b 100644 --- a/docs/build/html/_modules/quapy/classification/neural.html +++ b/docs/build/html/_modules/quapy/classification/neural.html @@ -1,95 +1,396 @@ + - - - - - quapy.classification.neural — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.classification.neural — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.classification.neural

-import os
-from abc import ABCMeta, abstractmethod
-from pathlib import Path
+import logging
+import os
+from abc import ABCMeta, abstractmethod
+from pathlib import Path
 
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from sklearn.metrics import accuracy_score, f1_score
-from torch.nn.utils.rnn import pad_sequence
-from tqdm import tqdm
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from sklearn.metrics import accuracy_score, f1_score
+from torch.nn.utils.rnn import pad_sequence
+from tqdm import tqdm
 
-import quapy as qp
-from quapy.data import LabelledCollection
-from quapy.util import EarlyStop
+import quapy as qp
+from quapy.data import LabelledCollection
+from quapy.util import EarlyStop
 
 
 

 [docs]
-class NeuralClassifierTrainer:
+class NeuralClassifierTrainer:
     """
     Trains a neural network for text classification.
 
@@ -107,7 +408,7 @@
         according to the evaluation in the held-out validation split (default '../checkpoint/classifier_net.dat')
     """
 
-    def __init__(self,
+    def __init__(self,
                  net: 'TextClassifierNet',
                  lr=1e-3,
                  weight_decay=0,
@@ -116,7 +417,7 @@
                  batch_size=64,
                  batch_size_test=512,
                  padding_length=300,
-                 device='cuda',
+                 device='cpu',
                  checkpointpath='../checkpoint/classifier_net.dat'):
 
         super().__init__()
@@ -137,12 +438,12 @@
         self.learner_hyperparams = self.net.get_params()
         self.checkpointpath = checkpointpath
 
-        print(f'[NeuralNetwork running on {device}]')
+        logging.getLogger(__name__).info(f'NeuralNetwork running on {device}')
         os.makedirs(Path(checkpointpath).parent, exist_ok=True)
 
 

 [docs]
-    def reset_net_params(self, vocab_size, n_classes):
+    def reset_net_params(self, vocab_size, n_classes):
         """Reinitialize the network parameters
 
         :param vocab_size: the size of the vocabulary
@@ -155,7 +456,7 @@
 
 

 [docs]
-    def get_params(self):
+    def get_params(self):
         """Get hyper-parameters for this estimator
 
         :return: a dictionary with parameter names mapped to their values
@@ -165,7 +466,7 @@
 
 

 [docs]
-    def set_params(self, **params):
+    def set_params(self, **params):
         """Set the parameters of this trainer and the learner it is training.
         In this current version, parameter names for the trainer and learner should
         be disjoint.
@@ -191,14 +492,14 @@
 
 
     @property
-    def device(self):
+    def device(self):
         """ Gets the device in which the network is allocated
 
         :return: device
         """
         return next(self.net.parameters()).device
 
-    def _train_epoch(self, data, status, pbar, epoch):
+    def _train_epoch(self, data, status, pbar, epoch):
         self.net.train()
         criterion = torch.nn.CrossEntropyLoss()
         losses, predictions, true_labels = [], [], []
@@ -218,7 +519,7 @@
             status["f1"] = f1_score(true_labels, predictions, average='macro')
             self.__update_progress_bar(pbar, epoch)
 
-    def _test_epoch(self, data, status, pbar, epoch):
+    def _test_epoch(self, data, status, pbar, epoch):
         self.net.eval()
         criterion = torch.nn.CrossEntropyLoss()
         losses, predictions, true_labels = [], [], []
@@ -236,7 +537,7 @@
             status["f1"] = f1_score(true_labels, predictions, average='macro')
             self.__update_progress_bar(pbar, epoch)
 
-    def __update_progress_bar(self, pbar, epoch):
+    def __update_progress_bar(self, pbar, epoch):
         pbar.set_description(f'[{self.net.__class__.__name__}] training epoch={epoch} '
                              f'tr-loss={self.status["tr"]["loss"]:.5f} '
                              f'tr-acc={100 * self.status["tr"]["acc"]:.2f}% '
@@ -248,7 +549,7 @@
 
 

 [docs]
-    def fit(self, instances, labels, val_split=0.3):
+    def fit(self, instances, labels, val_split=0.3):
         """
         Fits the model according to the given training data.
 
@@ -283,21 +584,22 @@
                 if self.early_stop.IMPROVED:
                     torch.save(self.net.state_dict(), checkpoint)
                 elif self.early_stop.STOP:
-                    print(f'training ended by patience exhasted; loading best model parameters in {checkpoint} '
-                          f'for epoch {self.early_stop.best_epoch}')
+                    logging.getLogger(__name__).info(
+                        f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
+                        f'for epoch {self.early_stop.best_epoch}')
                     self.net.load_state_dict(torch.load(checkpoint))
                     break
 
-        print('performing one training pass over the validation set...')
+        logging.getLogger(__name__).info('performing one training pass over the validation set...')
         self._train_epoch(valid_generator, self.status['tr'], pbar, epoch=0)
-        print('[done]')
+        logging.getLogger(__name__).info('done')
 
         return self

 
 
 

 [docs]
-    def predict(self, instances):
+    def predict(self, instances):
         """
         Predicts labels for the instances
 
@@ -310,7 +612,7 @@
 
 

 [docs]
-    def predict_proba(self, instances):
+    def predict_proba(self, instances):
         """
         Predicts posterior probabilities for the instances
 
@@ -329,7 +631,7 @@
 
 

 [docs]
-    def transform(self, instances):
+    def transform(self, instances):
         """
         Returns the embeddings of the instances
 
@@ -351,7 +653,7 @@
 
 

 [docs]
-class TorchDataset(torch.utils.data.Dataset):
+class TorchDataset(torch.utils.data.Dataset):
     """
     Transforms labelled instances into a Torch's :class:`torch.utils.data.DataLoader` object
 
@@ -359,19 +661,19 @@
     :param labels: array-like of shape `(n_samples, n_classes)` with the class labels
     """
 
-    def __init__(self, instances, labels=None):
+    def __init__(self, instances, labels=None):
         self.instances = instances
         self.labels = labels
 
-    def __len__(self):
+    def __len__(self):
         return len(self.instances)
 
-    def __getitem__(self, index):
+    def __getitem__(self, index):
         return {'doc': self.instances[index], 'label': self.labels[index] if self.labels is not None else None}
 
 

 [docs]
-    def asDataloader(self, batch_size, shuffle, pad_length, device):
+    def asDataloader(self, batch_size, shuffle, pad_length, device):
         """
         Converts the labelled collection into a Torch DataLoader with dynamic padding for
         the batch
@@ -384,7 +686,7 @@
         :param device: whether to allocate tensors in cpu or in cuda
         :return: a :class:`torch.utils.data.DataLoader` object
         """
-        def collate(batch):
+        def collate(batch):
             data = [torch.LongTensor(item['doc'][:pad_length]) for item in batch]
             data = pad_sequence(data, batch_first=True, padding_value=qp.environ['PAD_INDEX']).to(device)
             targets = [item['label'] for item in batch]
@@ -402,7 +704,7 @@
 
 

 [docs]
-class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta):
+class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta):
     """
     Abstract Text classifier (`torch.nn.Module`)
     """
@@ -410,7 +712,7 @@
 

 [docs]
     @abstractmethod
-    def document_embedding(self, x):
+    def document_embedding(self, x):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -425,7 +727,7 @@
 
 

 [docs]
-    def forward(self, x):
+    def forward(self, x):
         """Performs the forward pass.
 
         :param x: a batch of instances, typically generated by a torch's `DataLoader`
@@ -439,7 +741,7 @@
 
 

 [docs]
-    def dimensions(self):
+    def dimensions(self):
         """Gets the number of dimensions of the embedding space
 
         :return: integer
@@ -449,7 +751,7 @@
 
 

 [docs]
-    def predict_proba(self, x):
+    def predict_proba(self, x):
         """
         Predicts posterior probabilities for the instances in `x`
 
@@ -464,7 +766,7 @@
 
 

 [docs]
-    def xavier_uniform(self):
+    def xavier_uniform(self):
         """
         Performs Xavier initialization of the network parameters
         """
@@ -476,7 +778,7 @@
 

 [docs]
     @abstractmethod
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -486,7 +788,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -498,7 +800,7 @@
 
 

 [docs]
-class LSTMnet(TextClassifierNet):
+class LSTMnet(TextClassifierNet):
     """
     An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on
     Long Short Term Memory networks.
@@ -512,7 +814,7 @@
     :param drop_p: drop probability for dropout (default 0.5)
     """
 
-    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1,
+    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1,
                  drop_p=0.5):
 
         super().__init__()
@@ -534,7 +836,7 @@
         self.doc_embedder = torch.nn.Linear(hidden_size, self.dim)
         self.output = torch.nn.Linear(self.dim, n_classes)
 
-    def __init_hidden(self, set_size):
+    def __init_hidden(self, set_size):
         opt = self.hyperparams
         var_hidden = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size'])
         var_cell = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size'])
@@ -544,7 +846,7 @@
 
 

 [docs]
-    def document_embedding(self, x):
+    def document_embedding(self, x):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -563,7 +865,7 @@
 
 

 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -573,7 +875,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -585,7 +887,7 @@
 
 

 [docs]
-class CNNnet(TextClassifierNet):
+class CNNnet(TextClassifierNet):
     """
     An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on
     Convolutional Neural Networks.
@@ -602,7 +904,7 @@
     :param drop_p: drop probability for dropout (default 0.5)
     """
 
-    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100,
+    def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100,
                  kernel_heights=[3, 5, 7], stride=1, padding=0, drop_p=0.5):
         super(CNNnet, self).__init__()
 
@@ -627,7 +929,7 @@
         self.doc_embedder = torch.nn.Linear(len(kernel_heights) * hidden_size, self.dim)
         self.output = nn.Linear(self.dim, n_classes)
 
-    def __conv_block(self, input, conv_layer):
+    def __conv_block(self, input, conv_layer):
         conv_out = conv_layer(input)  # conv_out.size() = (batch_size, out_channels, dim, 1)
         activation = F.relu(conv_out.squeeze(3))  # activation.size() = (batch_size, out_channels, dim1)
         max_out = F.max_pool1d(activation, activation.size()[2]).squeeze(2)  # maxpool_out.size() = (batch_size, out_channels)
@@ -635,7 +937,7 @@
 
 

 [docs]
-    def document_embedding(self, input):
+    def document_embedding(self, input):
         """Embeds documents (i.e., performs the forward pass up to the
         next-to-last layer).
 
@@ -660,7 +962,7 @@
 
 

 [docs]
-    def get_params(self):
+    def get_params(self):
         """
         Get hyper-parameters for this estimator
 
@@ -670,7 +972,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         Return the size of the vocabulary
 
@@ -685,31 +987,75 @@
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.19.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/classification/svmperf.html b/docs/build/html/_modules/quapy/classification/svmperf.html index 959ad48..fb12a66 100644 --- a/docs/build/html/_modules/quapy/classification/svmperf.html +++ b/docs/build/html/_modules/quapy/classification/svmperf.html @@ -1,90 +1,391 @@ + - - - - - quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.classification.svmperf

-import random
-import shutil
-import subprocess
-import tempfile
-from os import remove, makedirs
-from os.path import join, exists
-from subprocess import PIPE, STDOUT
-import numpy as np
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.datasets import dump_svmlight_file
+import logging
+import random
+import shutil
+import subprocess
+import tempfile
+from os import remove, makedirs
+from os.path import join, exists
+from subprocess import PIPE, STDOUT
+import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.datasets import dump_svmlight_file
 
 
 

 [docs]
-class SVMperf(BaseEstimator, ClassifierMixin):
+class SVMperf(BaseEstimator, ClassifierMixin):
     """A wrapper for the `SVM-perf package <https://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html>`__ by Thorsten Joachims.
     When using losses for quantification, the source code has to be patched. See
     the `installation documentation <https://hlt-isti.github.io/QuaPy/build/html/Installation.html#svm-perf-with-quantification-oriented-losses>`__
@@ -106,31 +407,20 @@
     # losses with their respective codes in svm_perf implementation
     valid_losses = {'01':0, 'f1':1, 'kld':12, 'nkld':13, 'q':22, 'qacc':23, 'qf1':24, 'qgm':25, 'mae':26, 'mrae':27}
 
-    def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
-        assert exists(svmperf_base), f'path {svmperf_base} does not seem to point to a valid path'
+    def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
+        assert exists(svmperf_base), \
+            (f'path {svmperf_base} does not seem to point to a valid path;'
+             f'did you install svm-perf? '
+             f'see instructions in https://hlt-isti.github.io/QuaPy/manuals/explicit-loss-minimization.html')
         self.svmperf_base = svmperf_base
         self.C = C
         self.verbose = verbose
         self.loss = loss
         self.host_folder = host_folder
 
-    # def set_params(self, **parameters):
-    #     """
-    #     Set the hyper-parameters for svm-perf. Currently, only the `C` and `loss` parameters are supported
-    #
-    #     :param parameters: a `**kwargs` dictionary `{'C': <float>}`
-    #     """
-    #     assert sorted(list(parameters.keys())) == ['C', 'loss'], \
-    #         'currently, only the C and loss parameters are supported'
-    #     self.C = parameters.get('C', self.C)
-    #     self.loss = parameters.get('loss', self.loss)
-    #
-    # def get_params(self, deep=True):
-    #     return {'C': self.C, 'loss': self.loss}
-
 

 [docs]
-    def fit(self, X, y):
+    def fit(self, X, y):
         """
         Trains the SVM for the multivariate performance loss
 
@@ -153,8 +443,7 @@
         # this would allow to run parallel instances of predict
         random_code = 'svmperfprocess'+'-'.join(str(local_random.randint(0, 1000000)) for _ in range(5))
         if self.host_folder is None:
-            # tmp dir are removed after the fit terminates in multiprocessing...
-            self.tmpdir = tempfile.TemporaryDirectory(suffix=random_code).name
+            self.tmpdir = join(tempfile.gettempdir(), random_code)
         else:
             self.tmpdir = join(self.host_folder, '.' + random_code)
         makedirs(self.tmpdir, exist_ok=True)
@@ -166,21 +455,21 @@
 
         cmd = ' '.join([self.svmperf_learn, self.c_cmd, self.loss_cmd, traindat, self.model])
         if self.verbose:
-            print('[Running]', cmd)
-        p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT)
+            logging.getLogger(__name__).info(f'[Running] {cmd}')
+        p = subprocess.run(cmd.split(), stdout=PIPE, stderr=PIPE)
         if not exists(self.model):
-            print(p.stderr.decode('utf-8'))
+            logging.getLogger(__name__).error(p.stderr.decode('utf-8'))
         remove(traindat)
 
         if self.verbose:
-            print(p.stdout.decode('utf-8'))
+            logging.getLogger(__name__).info(p.stdout.decode('utf-8'))
 
         return self

 
 
 

 [docs]
-    def predict(self, X):
+    def predict(self, X):
         """
         Predicts labels for the instances `X`
 
@@ -195,7 +484,7 @@
 
 

 [docs]
-    def decision_function(self, X, y=None):
+    def decision_function(self, X, y=None):
         """
         Evaluate the decision function for the samples in `X`.
 
@@ -218,11 +507,11 @@
 
         cmd = ' '.join([self.svmperf_classify, testdat, self.model, predictions_path])
         if self.verbose:
-            print('[Running]', cmd)
+            logging.getLogger(__name__).info(f'[Running] {cmd}')
         p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT)
 
         if self.verbose:
-            print(p.stdout.decode('utf-8'))
+            logging.getLogger(__name__).info(p.stdout.decode('utf-8'))
 
         scores = np.loadtxt(predictions_path)
         remove(testdat)
@@ -231,38 +520,82 @@
         return scores

 
 
-    def __del__(self):
+    def __del__(self):
         if hasattr(self, 'tmpdir'):
             shutil.rmtree(self.tmpdir, ignore_errors=True)
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/data/base.html b/docs/build/html/_modules/quapy/data/base.html index e3a2e89..56c2697 100644 --- a/docs/build/html/_modules/quapy/data/base.html +++ b/docs/build/html/_modules/quapy/data/base.html @@ -1,91 +1,392 @@ + - - - - - quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ +

Source code for quapy.data.base

+import itertools
+from functools import cached_property
+from typing import Iterable
+
+import numpy as np
+from scipy.sparse import issparse
+from scipy.sparse import vstack
+from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
+from numpy.random import RandomState
+from quapy.functional import strprev
+from quapy.util import temp_seed
+import quapy.functional as F
 
 
 

 [docs]
-class LabelledCollection:
+class LabelledCollection:
     """
     A LabelledCollection is a set of objects each with a label attached to each of them. 
     This class implements several sampling routines and other utilities.
@@ -97,7 +398,7 @@
         (i.e., a prevalence of 0)
     """
 
-    def __init__(self, instances, labels, classes=None):
+    def __init__(self, instances, labels, classes=None):
         if issparse(instances):
             self.instances = instances
         elif isinstance(instances, list) and len(instances) > 0 and isinstance(instances[0], str):
@@ -106,21 +407,25 @@
         else:
             self.instances = np.asarray(instances)
         self.labels = np.asarray(labels)
-        n_docs = len(self)
         if classes is None:
-            self.classes_ = np.unique(self.labels)
-            self.classes_.sort()
+            self.classes_ = F.classes_from_labels(self.labels)
         else:
             self.classes_ = np.unique(np.asarray(classes))
             self.classes_.sort()
             if len(set(self.labels).difference(set(classes))) > 0:
                 raise ValueError(f'labels ({set(self.labels)}) contain values not included in classes_ ({set(classes)})')
-        self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_}
+        self._index = None
+
+    @property
+    def index(self):
+        if not hasattr(self, '_index') or self._index is None:
+            self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_}
+        return self._index
 
 

 [docs]
     @classmethod
-    def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
+    def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
         """
         Loads a labelled set of data and convert it into a :class:`LabelledCollection` instance. The function in charge
         of reading the instances must be specified. This function can be a custom one, or any of the reading functions
@@ -137,7 +442,7 @@
         return LabelledCollection(*loader_func(path, **loader_kwargs), classes)

 
 
-    def __len__(self):
+    def __len__(self):
         """
         Returns the length of this collection (number of labelled instances)
 
@@ -147,7 +452,7 @@
 
 

 [docs]
-    def prevalence(self):
+    def prevalence(self):
         """
         Returns the prevalence, or relative frequency, of the classes in the codeframe.
 
@@ -159,7 +464,7 @@
 
 

 [docs]
-    def counts(self):
+    def counts(self):
         """
         Returns the number of instances for each of the classes in the codeframe.
 
@@ -170,7 +475,7 @@
 
 
     @property
-    def n_classes(self):
+    def n_classes(self):
         """
         The number of classes
 
@@ -179,7 +484,16 @@
         return len(self.classes_)
 
     @property
-    def binary(self):
+    def n_instances(self):
+        """
+        The number of instances
+
+        :return: integer
+        """
+        return len(self.labels)
+
+    @property
+    def binary(self):
         """
         Returns True if the number of classes is 2
 
@@ -189,12 +503,11 @@
 
 

 [docs]
-    def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
+    def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
         """
         Returns an index to be used to extract a random sample of desired size and desired prevalence values. If the
         prevalence values are not specified, then returns the index of a uniform sampling.
-        For each class, the sampling is drawn with replacement if the requested prevalence is larger than
-        the actual prevalence of the class, or without replacement otherwise.
+        For each class, the sampling is drawn with replacement.
 
         :param size: integer, the requested size
         :param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -209,7 +522,7 @@
         if len(prevs) == self.n_classes - 1:
             prevs = prevs + (1 - sum(prevs),)
         assert len(prevs) == self.n_classes, 'unexpected number of prevalences'
-        assert sum(prevs) == 1, f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
+        assert np.isclose(sum(prevs), 1), f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
 
         # Decide how many instances should be taken for each class in order to satisfy the requested prevalence
         # accurately, and the number of instances in the sample (exactly). If int(size * prevs[i]) (which is
@@ -238,7 +551,7 @@
             for class_, n_requested in n_requests.items():
                 n_candidates = len(self.index[class_])
                 index_sample = self.index[class_][
-                    np.random.choice(n_candidates, size=n_requested, replace=(n_requested > n_candidates))
+                    np.random.choice(n_candidates, size=n_requested, replace=True)
                 ] if n_requested > 0 else []
 
                 indexes_sample.append(index_sample)
@@ -253,11 +566,10 @@
 
 

 [docs]
-    def uniform_sampling_index(self, size, random_state=None):
+    def uniform_sampling_index(self, size, random_state=None):
         """
         Returns an index to be used to extract a uniform sample of desired size. The sampling is drawn
-        with replacement if the requested size is greater than the number of instances, or without replacement
-        otherwise.
+        with replacement.
 
         :param size: integer, the size of the uniform sample
         :param random_state: if specified, guarantees reproducibility of the split.
@@ -267,16 +579,15 @@
             ng = RandomState(seed=random_state)
         else:
             ng = np.random
-        return ng.choice(len(self), size, replace=size > len(self))

+        return ng.choice(len(self), size, replace=True)

 
 
 

 [docs]
-    def sampling(self, size, *prevs, shuffle=True, random_state=None):
+    def sampling(self, size, *prevs, shuffle=True, random_state=None):
         """
         Return a random sample (an instance of :class:`LabelledCollection`) of desired size and desired prevalence
-        values. For each class, the sampling is drawn without replacement if the requested prevalence is larger than
-        the actual prevalence of the class, or with replacement otherwise.
+        values. For each class, the sampling is drawn with replacement.
 
         :param size: integer, the requested size
         :param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -293,11 +604,10 @@
 
 

 [docs]
-    def uniform_sampling(self, size, random_state=None):
+    def uniform_sampling(self, size, random_state=None):
         """
         Returns a uniform sample (an instance of :class:`LabelledCollection`) of desired size. The sampling is drawn
-        with replacement if the requested size is greater than the number of instances, or without replacement
-        otherwise.
+        with replacement.
 
         :param size: integer, the requested size
         :param random_state: if specified, guarantees reproducibility of the split.
@@ -309,7 +619,7 @@
 
 

 [docs]
-    def sampling_from_index(self, index):
+    def sampling_from_index(self, index):
         """
         Returns an instance of :class:`LabelledCollection` whose elements are sampled from this collection using the
         index.
@@ -324,7 +634,7 @@
 
 

 [docs]
-    def split_stratified(self, train_prop=0.6, random_state=None):
+    def split_stratified(self, train_prop=0.6, random_state=None):
         """
         Returns two instances of :class:`LabelledCollection` split with stratification from this collection, at desired
         proportion.
@@ -336,17 +646,17 @@
         :return: two instances of :class:`LabelledCollection`, the first one with `train_prop` elements, and the
             second one with `1-train_prop` elements
         """
-        tr_docs, te_docs, tr_labels, te_labels = train_test_split(
+        tr_X, te_X, tr_y, te_y = train_test_split(
             self.instances, self.labels, train_size=train_prop, stratify=self.labels, random_state=random_state
         )
-        training = LabelledCollection(tr_docs, tr_labels, classes=self.classes_)
-        test = LabelledCollection(te_docs, te_labels, classes=self.classes_)
+        training = LabelledCollection(tr_X, tr_y, classes=self.classes_)
+        test = LabelledCollection(te_X, te_y, classes=self.classes_)
         return training, test

 
 
 

 [docs]
-    def split_random(self, train_prop=0.6, random_state=None):
+    def split_random(self, train_prop=0.6, random_state=None):
         """
         Returns two instances of :class:`LabelledCollection` split randomly from this collection, at desired
         proportion.
@@ -373,7 +683,7 @@
         return training, test

 
 
-    def __add__(self, other):
+    def __add__(self, other):
         """
         Returns a new :class:`LabelledCollection` as the union of this collection with another collection.
         Both labelled collections must have the same classes.
@@ -389,7 +699,7 @@
 

 [docs]
     @classmethod
-    def join(cls, *args: Iterable['LabelledCollection']):
+    def join(cls, *args: Iterable['LabelledCollection']):
         """
         Returns a new :class:`LabelledCollection` as the union of the collections given in input.
 
@@ -425,12 +735,23 @@
         else:
             raise NotImplementedError('unsupported operation for collection types')
         labels = np.concatenate([lc.labels for lc in args])
-        classes = np.unique(labels).sort()
+        # union of each collection's own classes_, so a class declared but absent from
+        # this particular join (e.g. an empty fold) is preserved at zero prevalence
+        classes = np.unique(np.concatenate([lc.classes_ for lc in args]))
         return LabelledCollection(instances, labels, classes=classes)

 
 
     @property
-    def Xy(self):
+    def classes(self):
+        """
+        Gets an array-like with the classes used in this collection
+
+        :return: array-like
+        """
+        return self.classes_
+
+    @property
+    def Xy(self):
         """
         Gets the instances and labels. This is useful when working with `sklearn` estimators, e.g.:
 
@@ -441,7 +762,7 @@
         return self.instances, self.labels
 
     @property
-    def Xp(self):
+    def Xp(self):
         """
         Gets the instances and the true prevalence. This is useful when implementing evaluation protocols from
         a :class:`LabelledCollection` object.
@@ -451,7 +772,7 @@
         return self.instances, self.prevalence()
 
     @property
-    def X(self):
+    def X(self):
         """
         An alias to self.instances
 
@@ -460,7 +781,7 @@
         return self.instances
 
     @property
-    def y(self):
+    def y(self):
         """
         An alias to self.labels
 
@@ -469,7 +790,7 @@
         return self.labels
 
     @property
-    def p(self):
+    def p(self):
         """
         An alias to self.prevalence()
 
@@ -480,7 +801,7 @@
 
 

 [docs]
-    def stats(self, show=True):
+    def stats(self, show=True):
         """
         Returns (and eventually prints) a dictionary with some stats of this collection. E.g.,:
 
@@ -515,7 +836,7 @@
 
 

 [docs]
-    def kFCV(self, nfolds=5, nrepeats=1, random_state=None):
+    def kFCV(self, nfolds=5, nrepeats=1, random_state=None):
         """
         Generator of stratified folds to be used in k-fold cross validation.
 
@@ -529,13 +850,18 @@
             train = self.sampling_from_index(train_index)
             test = self.sampling_from_index(test_index)
             yield train, test

-

+
+
+    def __repr__(self):
+        repr=f'<{self.n_instances} instances (dtype={type(self.instances[0])}), '
+        repr+=f'n_classes={self.n_classes} {self.classes_}, prevalence={F.strprev(self.prevalence())}>'
+        return repr

 
 
 
 

 [docs]
-class Dataset:
+class Dataset:
     """
     Abstraction of training and test :class:`LabelledCollection` objects.
 
@@ -545,7 +871,7 @@
     :param name: a string representing the name of the dataset
     """
 
-    def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''):
+    def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''):
         assert set(training.classes_) == set(test.classes_), 'incompatible labels in training and test collections'
         self.training = training
         self.test = test
@@ -555,7 +881,7 @@
 

 [docs]
     @classmethod
-    def SplitStratified(cls, collection: LabelledCollection, train_size=0.6):
+    def SplitStratified(cls, collection: LabelledCollection, train_size=0.6):
         """
         Generates a :class:`Dataset` from a stratified split of a :class:`LabelledCollection` instance.
         See :meth:`LabelledCollection.split_stratified`
@@ -568,7 +894,7 @@
 
 
     @property
-    def classes_(self):
+    def classes_(self):
         """
         The classes according to which the training collection is labelled
 
@@ -577,7 +903,7 @@
         return self.training.classes_
 
     @property
-    def n_classes(self):
+    def n_classes(self):
         """
         The number of classes according to which the training collection is labelled
 
@@ -586,7 +912,7 @@
         return self.training.n_classes
 
     @property
-    def binary(self):
+    def binary(self):
         """
         Returns True if the training collection is labelled according to two classes
 
@@ -597,7 +923,7 @@
 

 [docs]
     @classmethod
-    def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs):
+    def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs):
         """
         Loads a training and a test labelled set of data and convert it into a :class:`Dataset` instance.
         The function in charge of reading the instances must be specified. This function can be a custom one, or any of
@@ -619,7 +945,7 @@
 
 
     @property
-    def vocabulary_size(self):
+    def vocabulary_size(self):
         """
         If the dataset is textual, and the vocabulary was indicated, returns the size of the vocabulary
 
@@ -628,7 +954,7 @@
         return len(self.vocabulary)
 
     @property
-    def train_test(self):
+    def train_test(self):
         """
         Alias to `self.training` and `self.test`
 
@@ -639,7 +965,7 @@
 
 

 [docs]
-    def stats(self, show=True):
+    def stats(self, show=True):
         """
         Returns (and eventually prints) a dictionary with some stats of this dataset. E.g.,:
 
@@ -666,7 +992,7 @@
 

 [docs]
     @classmethod
-    def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0):
+    def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0):
         """
         Generator of stratified folds to be used in k-fold cross validation. This function is only a wrapper around
         :meth:`LabelledCollection.kFCV` that returns :class:`Dataset` instances made of training and test folds.
@@ -683,7 +1009,7 @@
 
 

 [docs]
-    def reduce(self, n_train=100, n_test=100):
+    def reduce(self, n_train=100, n_test=100, random_state=None):
         """
         Reduce the number of instances in place for quick experiments. Preserves the prevalence of each set.
 
@@ -691,38 +1017,93 @@
         :param n_test: number of test documents to keep (default 100)
         :return: self
         """
-        self.training = self.training.sampling(n_train, *self.training.prevalence())
-        self.test = self.test.sampling(n_test, *self.test.prevalence())
+        self.training = self.training.sampling(
+            n_train,
+            *self.training.prevalence(),
+            random_state = random_state
+        )
+        self.test = self.test.sampling(
+            n_test,
+            *self.test.prevalence(),
+            random_state = random_state
+        )
         return self

-

+
+
+    def __repr__(self):
+        return f'training={self.training}; test={self.test}'
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/data/datasets.html b/docs/build/html/_modules/quapy/data/datasets.html index b910036..b0fb218 100644 --- a/docs/build/html/_modules/quapy/data/datasets.html +++ b/docs/build/html/_modules/quapy/data/datasets.html @@ -1,129 +1,475 @@ + - - - - - quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.data.datasets

-
[docs]def warn(*args, **kwargs):
-    pass

-import warnings
-warnings.warn = warn
-import os
-import zipfile
-from os.path import join
-import pandas as pd
-from ucimlrepo import fetch_ucirepo
-from quapy.data.base import Dataset, LabelledCollection
-from quapy.data.preprocessing import text2tfidf, reduce_columns
-from quapy.data.reader import *
-from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
+import logging
+import os
+from contextlib import contextmanager
+import zipfile
+from os.path import join
+import pandas as pd
+from quapy.data.base import Dataset, LabelledCollection
+from quapy.data.preprocessing import text2tfidf, reduce_columns
+from quapy.data.preprocessing import standardize as standardizer
+from quapy.data.reader import *
+from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
+from sklearn.preprocessing import StandardScaler
+
+
+def _fetch_ucirepo(*args, **kwargs):
+    try:
+        from ucimlrepo import fetch_ucirepo
+    except ImportError as exc:
+        raise ImportError(
+            "UCI dataset fetching requires the optional 'ucimlrepo' package. "
+            "Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset."
+        ) from exc
+    return fetch_ucirepo(*args, **kwargs)
 
 
 REVIEWS_SENTIMENT_DATASETS = ['hp', 'kindle', 'imdb']
-TWITTER_SENTIMENT_DATASETS_TEST = ['gasp', 'hcr', 'omd', 'sanders',
-                              'semeval13', 'semeval14', 'semeval15', 'semeval16',
-                              'sst', 'wa', 'wb']
-TWITTER_SENTIMENT_DATASETS_TRAIN = ['gasp', 'hcr', 'omd', 'sanders',
-                                 'semeval', 'semeval16',
-                                 'sst', 'wa', 'wb']
-UCI_BINARY_DATASETS = ['acute.a', 'acute.b',
-                'balance.1', 'balance.2', 'balance.3',
-                'breast-cancer',
-                'cmc.1', 'cmc.2', 'cmc.3',
-                'ctg.1', 'ctg.2', 'ctg.3',
-                       #'diabetes', # <-- I haven't found this one...
-                'german',
-                'haberman',
-                'ionosphere',
-                'iris.1', 'iris.2', 'iris.3',
-                'mammographic',
-                       'pageblocks.5',
-                       #'phoneme', # <-- I haven't found this one...
-                       'semeion',
-                       'sonar',
-                       'spambase',
-                       'spectf',
-                       'tictactoe',
-                       'transfusion',
-                       'wdbc',
-                       'wine.1', 'wine.2', 'wine.3',
-                       'wine-q-red', 'wine-q-white',
-                       'yeast']
 
-UCI_MULTICLASS_DATASETS = ['dry-bean',
-                           'wine-quality',
-                           'academic-success',
-                           'digits',
-                           'letter']
+TWITTER_SENTIMENT_DATASETS_TEST = [
+    'gasp', 'hcr', 'omd', 'sanders',
+    'semeval13', 'semeval14', 'semeval15', 'semeval16',
+    'sst', 'wa', 'wb',
+]
 
-LEQUA2022_TASKS = ['T1A', 'T1B', 'T2A', 'T2B']
+TWITTER_SENTIMENT_DATASETS_TRAIN = [
+    'gasp', 'hcr', 'omd', 'sanders',
+    'semeval', 'semeval16',
+    'sst', 'wa', 'wb',
+]
+
+UCI_BINARY_DATASETS = [
+    #'acute.a', 'acute.b',
+    'balance.1', 
+    #'balance.2', 
+    'balance.3',
+    'breast-cancer',
+    'cmc.1', 'cmc.2', 'cmc.3',
+    'ctg.1', 'ctg.2', 'ctg.3',
+    #'diabetes', # <-- I haven't found this one...
+    'german',
+    'haberman',
+    'ionosphere',
+    'iris.1', 'iris.2', 'iris.3',
+    'mammographic',
+    'pageblocks.5',
+    #'phoneme', # <-- I haven't found this one...
+    'semeion',
+    'sonar',
+    'spambase',
+    'spectf',
+    'tictactoe',
+    'transfusion',
+    'wdbc',
+    'wine.1', 'wine.2', 'wine.3',
+    'wine-q-red',
+    'wine-q-white',
+    'yeast',
+]
+
+UCI_MULTICLASS_DATASETS = [
+    'dry-bean',
+    'wine-quality',
+    'academic-success',
+    'digits',
+    'letter',
+    'abalone',
+    'obesity',
+    'nursery',
+    'yeast',
+    'hand_digits',
+    'satellite',
+    'shuttle',
+    'cmc',
+    'isolet',
+    'waveform-v1',
+    'molecular',
+    'poker_hand',
+    'connect-4',
+    'mhr',
+    'chess',
+    'page_block',
+    'phishing',
+    'image_seg',
+    'hcv',
+]
+
+LEQUA2022_VECTOR_TASKS = ['T1A', 'T1B']
+LEQUA2022_TEXT_TASKS = ['T2A', 'T2B']
+LEQUA2022_TASKS = LEQUA2022_VECTOR_TASKS + LEQUA2022_TEXT_TASKS
+
+LEQUA2024_TASKS = ['T1', 'T2', 'T3', 'T4']
 
 _TXA_SAMPLE_SIZE = 250
 _TXB_SAMPLE_SIZE = 1000
@@ -139,12 +485,25 @@
     'multiclass': _TXB_SAMPLE_SIZE
 }
 
+LEQUA2024_SAMPLE_SIZE = {
+    'T1': 250,
+    'T2': 1000,
+    'T3': 200,
+    'T4': 250,
+}
 
-
[docs]def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset:
+IMAGE_DATASETS=['cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist']
+IMAGE_EMBEDDINGS=['features', 'logits', 'predictions']
+
+
+

+[docs]
+def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset:
     """
     Loads a Reviews dataset as a Dataset instance, as used in
     `Esuli, A., Moreo, A., and Sebastiani, F. "A recurrent neural network for sentiment quantification."
-    Proceedings of the 27th ACM International Conference on Information and Knowledge Management. 2018. <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_.
+    Proceedings of the 27th ACM International Conference on Information and Knowledge Management. 2018.
+    <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_.
     The list of valid dataset names can be accessed in `quapy.data.datasets.REVIEWS_SENTIMENT_DATASETS`
 
     :param dataset_name: the name of the dataset: valid ones are 'hp', 'kindle', 'imdb'
@@ -186,7 +545,10 @@
     return data

 
 
-
[docs]def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset:
+
+

+[docs]
+def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset:
     """
     Loads a Twitter dataset as a :class:`quapy.data.base.Dataset` instance, as used in:
     `Gao, W., Sebastiani, F.: From classification to quantification in tweet sentiment analysis.
@@ -228,8 +590,9 @@
     if dataset_name in {'semeval13', 'semeval14', 'semeval15'}:
         trainset_name = 'semeval'
         testset_name  = 'semeval' if for_model_selection else dataset_name
-        print(f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common "
-              f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}")
+        logging.getLogger(__name__).info(
+            f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common "
+            f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}")
     else:
         if dataset_name == 'semeval' and for_model_selection==False:
             raise ValueError('dataset "semeval" can only be used for model selection. '
@@ -260,7 +623,10 @@
     return data

 
 
-
[docs]def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, verbose=False) -> Dataset:
+
+

+[docs]
+def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, standardize=True, verbose=False) -> Dataset:
     """
     Loads a UCI dataset as an instance of :class:`quapy.data.base.Dataset`, as used in
     `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).
@@ -278,14 +644,23 @@
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
     :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,
+        standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the
+        training portion of the data.
     :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets
     :return: a :class:`quapy.data.base.Dataset` instance
     """
     data = fetch_UCIBinaryLabelledCollection(dataset_name, data_home, verbose)
-    return Dataset(*data.split_stratified(1 - test_split, random_state=0))

+    dataset = Dataset(*data.split_stratified(1 - test_split, random_state=0), name=dataset_name)
+    if standardize:
+        dataset = standardizer(dataset)
+    return dataset

 
 
-
[docs]def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, verbose=False) -> LabelledCollection:
+
+

+[docs]
+def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize=True, verbose=False) -> LabelledCollection:
     """
     Loads a UCI collection as an instance of :class:`quapy.data.base.LabelledCollection`, as used in
     `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).
@@ -301,7 +676,7 @@
 
     >>> import quapy as qp
     >>> collection = qp.datasets.fetch_UCIBinaryLabelledCollection("yeast")
-    >>> for data in qp.train.Dataset.kFCV(collection, nfolds=5, nrepeats=2):
+    >>> for data in qp.datasets.Dataset.kFCV(collection, nfolds=5, nrepeats=2):
     >>>     ...
 
     The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_DATASETS`
@@ -309,327 +684,312 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). 
     :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets
     :return: a :class:`quapy.data.base.LabelledCollection` instance
     """
-
-    assert dataset_name in UCI_BINARY_DATASETS, \
-        f'Name {dataset_name} does not match any known dataset from the UCI Machine Learning datasets repository. ' \
-        f'Valid ones are {UCI_BINARY_DATASETS}'
+    assert dataset_name in UCI_BINARY_DATASETS, (
+        f"Name {dataset_name} does not match any known dataset from the UCI Machine Learning datasets repository. "
+        f"Valid ones are {UCI_BINARY_DATASETS}"
+    )
     if data_home is None:
         data_home = get_quapy_home()
 
-    dataset_fullname = {
-        'acute.a': 'Acute Inflammations (urinary bladder)',
-        'acute.b': 'Acute Inflammations (renal pelvis)',
-        'balance.1': 'Balance Scale Weight & Distance Database (left)',
-        'balance.2': 'Balance Scale Weight & Distance Database (balanced)',
-        'balance.3': 'Balance Scale Weight & Distance Database (right)',
-        'breast-cancer':  'Breast Cancer Wisconsin (Original)',
-        'cmc.1': 'Contraceptive Method Choice (no use)',
-        'cmc.2': 'Contraceptive Method Choice (long term)',
-        'cmc.3': 'Contraceptive Method Choice (short term)',
-        'ctg.1': 'Cardiotocography Data Set (normal)',
-        'ctg.2': 'Cardiotocography Data Set (suspect)',
-        'ctg.3': 'Cardiotocography Data Set (pathologic)',
-        'german': 'Statlog German Credit Data',
-        'haberman': "Haberman's Survival Data",
-        'ionosphere': 'Johns Hopkins University Ionosphere DB',
-        'iris.1': 'Iris Plants Database(x)',
-        'iris.2': 'Iris Plants Database(versicolour)',
-        'iris.3': 'Iris Plants Database(virginica)',
-        'mammographic': 'Mammographic Mass',
-        'pageblocks.5': 'Page Blocks Classification (5)',
-        'semeion': 'Semeion Handwritten Digit (8)',
-        'sonar': 'Sonar, Mines vs. Rocks',
-        'spambase': 'Spambase Data Set',
-        'spectf': 'SPECTF Heart Data',
-        'tictactoe': 'Tic-Tac-Toe Endgame Database',
-        'transfusion': 'Blood Transfusion Service Center Data Set',
-        'wdbc': 'Wisconsin Diagnostic Breast Cancer',
-        'wine.1': 'Wine Recognition Data (1)',
-        'wine.2': 'Wine Recognition Data (2)',
-        'wine.3': 'Wine Recognition Data (3)',
-        'wine-q-red': 'Wine Quality Red (6-10)',
-        'wine-q-white': 'Wine Quality White (6-10)',
-        'yeast': 'Yeast',
+    # mapping bewteen dataset names and UCI api ids
+    identifiers = {
+        "acute.a": 184,
+        "acute.b": 184,
+        "balance.1": 12,
+        "balance.2": 12,
+        "balance.3": 12,
+        "breast-cancer": 15,
+        "cmc.1": 30,
+        "cmc.2": 30,
+        "cmc.3": 30,
+        # "ctg.1": ,  # not python importable
+        # "ctg.2": ,  # not python importable
+        # "ctg.3": ,  # not python importable
+        # "german": ,  # not python importable
+        "haberman": 43,
+        "ionosphere": 52,
+        "iris.1": 53,
+        "iris.2": 53,
+        "iris.3": 53,
+        "mammographic": 161,
+        "pageblocks.5": 78,
+        # "semeion": ,  # not python importable
+        "sonar": 151,
+        "spambase": 94,
+        "spectf": 96,
+        "tictactoe": 101,
+        "transfusion": 176,
+        "wdbc": 17,
+        "wine.1": 109,
+        "wine.2": 109,
+        "wine.3": 109,
+        "wine-q-red": 186,
+        "wine-q-white": 186,
+        "yeast": 110,
     }
 
-    # the identifier is an alias for the dataset group, it's part of the url data-folder, and is the name we use
-    # to download the raw dataset
-    identifier_map = {
-        'acute.a': 'acute',
-        'acute.b': 'acute',
-        'balance.1': 'balance-scale',
-        'balance.2': 'balance-scale',
-        'balance.3': 'balance-scale',
-        'breast-cancer': 'breast-cancer-wisconsin',
-        'cmc.1': 'cmc',
-        'cmc.2': 'cmc',
-        'cmc.3': 'cmc',
-        'ctg.1': '00193',
-        'ctg.2': '00193',
-        'ctg.3': '00193',
-        'german': 'statlog/german',
-        'haberman': 'haberman',
-        'ionosphere': 'ionosphere',
-        'iris.1': 'iris',
-        'iris.2': 'iris',
-        'iris.3': 'iris',
-        'mammographic': 'mammographic-masses',
-        'pageblocks.5': 'page-blocks',
-        'semeion': 'semeion',
-        'sonar': 'undocumented/connectionist-bench/sonar',
-        'spambase': 'spambase',
-        'spectf': 'spect',
-        'tictactoe': 'tic-tac-toe',
-        'transfusion': 'blood-transfusion',
-        'wdbc': 'breast-cancer-wisconsin',
-        'wine-q-red': 'wine-quality',
-        'wine-q-white': 'wine-quality',
-        'wine.1': 'wine',
-        'wine.2': 'wine',
-        'wine.3': 'wine',
-        'yeast': 'yeast',
+    # mapping between dataset names and dataset groups
+    groups = {
+        "acute.a": "acute",
+        "acute.b": "acute",
+        "balance.1": "balance",
+        "balance.2": "balance",
+        "balance.3": "balance",
+        "breast-cancer": "breast-cancer",
+        "cmc.1": "cmc",
+        "cmc.2": "cmc",
+        "cmc.3": "cmc",
+        "ctg.1": "ctg",
+        "ctg.2": "ctg",
+        "ctg.3": "ctg",
+        "german": "german",
+        "haberman": "haberman",
+        "ionosphere": "ionosphere",
+        "iris.1": "iris",
+        "iris.2": "iris",
+        "iris.3": "iris",
+        "mammographic": "mammographic",
+        "pageblocks.5": "pageblocks",
+        "semeion": "semeion",
+        "sonar": "sonar",
+        "spambase": "spambase",
+        "spectf": "spectf",
+        "tictactoe": "tictactoe",
+        "transfusion": "transfusion",
+        "wdbc": "wdbc",
+        "wine-q-red": "wine-quality",
+        "wine-q-white": "wine-quality",
+        "wine.1": "wine",
+        "wine.2": "wine",
+        "wine.3": "wine",
+        "yeast": "yeast",
     }
 
-    # the filename is the name of the file within the data_folder indexed by the identifier
-    file_name = {
-        'acute': 'diagnosis.data',
-        '00193': 'CTG.xls',
-        'statlog/german': 'german.data-numeric',
-        'mammographic-masses': 'mammographic_masses.data',
-        'page-blocks': 'page-blocks.data.Z',
-        'undocumented/connectionist-bench/sonar': 'sonar.all-data',
-        'spect': ['SPECTF.train', 'SPECTF.test'],
-        'blood-transfusion': 'transfusion.data',
-        'wine-quality': ['winequality-red.csv', 'winequality-white.csv'],
-        'breast-cancer-wisconsin': 'breast-cancer-wisconsin.data' if dataset_name=='breast-cancer' else 'wdbc.data'
+    # mapping between dataset short names and full names
+    full_names = {
+        "acute.a": "Acute Inflammations (urinary bladder)",
+        "acute.b": "Acute Inflammations (renal pelvis)",
+        "balance.1": "Balance Scale Weight & Distance Database (left)",
+        "balance.2": "Balance Scale Weight & Distance Database (balanced)",
+        "balance.3": "Balance Scale Weight & Distance Database (right)",
+        "breast-cancer": "Breast Cancer Wisconsin (Original)",
+        "cmc.1": "Contraceptive Method Choice (no use)",
+        "cmc.2": "Contraceptive Method Choice (long term)",
+        "cmc.3": "Contraceptive Method Choice (short term)",
+        "ctg.1": "Cardiotocography Data Set (normal)",
+        "ctg.2": "Cardiotocography Data Set (suspect)",
+        "ctg.3": "Cardiotocography Data Set (pathologic)",
+        "german": "Statlog German Credit Data",
+        "haberman": "Haberman's Survival Data",
+        "ionosphere": "Johns Hopkins University Ionosphere DB",
+        "iris.1": "Iris Plants Database(x)",
+        "iris.2": "Iris Plants Database(versicolour)",
+        "iris.3": "Iris Plants Database(virginica)",
+        "mammographic": "Mammographic Mass",
+        "pageblocks.5": "Page Blocks Classification (5)",
+        "semeion": "Semeion Handwritten Digit (8)",
+        "sonar": "Sonar, Mines vs. Rocks",
+        "spambase": "Spambase Data Set",
+        "spectf": "SPECTF Heart Data",
+        "tictactoe": "Tic-Tac-Toe Endgame Database",
+        "transfusion": "Blood Transfusion Service Center Data Set",
+        "wdbc": "Wisconsin Diagnostic Breast Cancer",
+        "wine.1": "Wine Recognition Data (1)",
+        "wine.2": "Wine Recognition Data (2)",
+        "wine.3": "Wine Recognition Data (3)",
+        "wine-q-red": "Wine Quality Red (6-10)",
+        "wine-q-white": "Wine Quality White (6-10)",
+        "yeast": "Yeast",
     }
 
-    # the filename containing the dataset description (if any)
-    desc_name = {
-        'acute': 'diagnosis.names',
-        '00193': None,
-        'statlog/german': 'german.doc',
-        'mammographic-masses': 'mammographic_masses.names',
-        'undocumented/connectionist-bench/sonar': 'sonar.names',
-        'spect': 'SPECTF.names',
-        'blood-transfusion': 'transfusion.names',
-        'wine-quality': 'winequality.names',
-        'breast-cancer-wisconsin': 'breast-cancer-wisconsin.names' if dataset_name == 'breast-cancer' else 'wdbc.names'
+    # mapping between dataset names and values of positive class
+    pos_class = {
+        "acute.a": "yes",
+        "acute.b": "yes",
+        "balance.1": "L",
+        "balance.2": "B",
+        "balance.3": "R",
+        "breast-cancer": 2,
+        "cmc.1": 1,
+        "cmc.2": 2,
+        "cmc.3": 3,
+        "ctg.1": 1,  # 1==Normal
+        "ctg.2": 2,  # 2==Suspect
+        "ctg.3": 3,  # 3==Pathologic
+        "german": 1,
+        "haberman": 2,
+        "ionosphere": "b",
+        "iris.1": "Iris-setosa",  # 1==Setosa
+        "iris.2": "Iris-versicolor",  # 2==Versicolor
+        "iris.3": "Iris-virginica",  # 3==Virginica
+        "mammographic": 1,
+        "pageblocks.5": 5,  # 5==block "graphic"
+        "semeion": 1,
+        "sonar": "R",
+        "spambase": 1,
+        "spectf": 0,
+        "tictactoe": "negative",
+        "transfusion": 1,
+        "wdbc": "M",
+        "wine.1": 1,
+        "wine.2": 2,
+        "wine.3": 3,
+        "wine-q-red": 1,
+        "wine-q-white": 1,
+        "yeast": "NUC",
     }
 
-    identifier = identifier_map[dataset_name]
-    filename = file_name.get(identifier, f'{identifier}.data')
-    descfile = desc_name.get(identifier, f'{identifier}.names')
-    fullname = dataset_fullname[dataset_name]
-
-    URL = f'http://archive.ics.uci.edu/ml/machine-learning-databases/{identifier}'
-    data_dir = join(data_home, 'uci_datasets', identifier)
-    if isinstance(filename, str):  # filename could be a list of files, in which case it will be processed later
-        data_path = join(data_dir, filename)
-        download_file_if_not_exists(f'{URL}/{filename}', data_path)
-
-    if descfile:
-        try:
-            download_file_if_not_exists(f'{URL}/{descfile}', f'{data_dir}/{descfile}')
-            if verbose:
-                print(open(f'{data_dir}/{descfile}', 'rt').read())
-        except Exception:
-            print('could not read the description file')
-    elif verbose:
-        print('no file description available')
+    identifier = identifiers.get(dataset_name, None)
+    dataset_group = groups[dataset_name]
+    fullname = full_names[dataset_name]
 
     if verbose:
-        print(f'Loading {dataset_name} ({fullname})')
-    if identifier == 'acute':
-        df = pd.read_csv(data_path, header=None, encoding='utf-16', sep='\t')
+        print(f"Loading UCI Binary {dataset_name} ({fullname})")
 
-        df[0] = df[0].apply(lambda x: float(x.replace(',', '.'))).astype(float, copy=False)
-        [_df_replace(df, col) for col in range(1, 6)]
-        X = df.loc[:, 0:5].values
-        if dataset_name == 'acute.a':
-            y = binarize(df[6], pos_class='yes')
-        elif dataset_name == 'acute.b':
-            y = binarize(df[7], pos_class='yes')
+    file = join(data_home, "uci_datasets", dataset_group + ".pkl")
 
-    if identifier == 'balance-scale':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        if dataset_name == 'balance.1':
-            y = binarize(df[0], pos_class='L')
-        elif dataset_name == 'balance.2':
-            y = binarize(df[0], pos_class='B')
-        elif dataset_name == 'balance.3':
-            y = binarize(df[0], pos_class='R')
-        X = df.loc[:, 1:].astype(float).values
+    @contextmanager
+    def download_tmp_file(url_group: str, filename: str):
+        """
+        Download a data file for a group of datasets temporarely.
+        When used as a context, the file is removed once the context exits.
 
-    if identifier == 'breast-cancer-wisconsin' and dataset_name=='breast-cancer':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        Xy = df.loc[:, 1:10]
-        Xy[Xy=='?']=np.nan
-        Xy = Xy.dropna(axis=0)
-        X = Xy.loc[:, 1:9]
-        X = X.astype(float).values
-        y = binarize(Xy[10], pos_class=2)
-
-    if identifier == 'breast-cancer-wisconsin' and dataset_name=='wdbc':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.loc[:, 2:32].astype(float).values
-        y = df[1].values
-        y = binarize(y, pos_class='M')
-
-    if identifier == 'cmc':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.loc[:, 0:8].astype(float).values
-        y = df[9].astype(int).values
-        if dataset_name == 'cmc.1':
-            y = binarize(y, pos_class=1)
-        elif dataset_name == 'cmc.2':
-            y = binarize(y, pos_class=2)
-        elif dataset_name == 'cmc.3':
-            y = binarize(y, pos_class=3)
-
-    if identifier == '00193':
-        df = pd.read_excel(data_path, sheet_name='Data', skipfooter=3)
-        df = df[list(range(1,24))] # select columns numbered (number 23 is the target label)
-        # replaces the header with the first row
-        new_header = df.iloc[0]  # grab the first row for the header
-        df = df[1:]  # take the data less the header row
-        df.columns = new_header  # set the header row as the df header
-        X = df.iloc[:, 0:22].astype(float).values
-        y = df['NSP'].astype(int).values
-        if dataset_name == 'ctg.1':
-            y = binarize(y, pos_class=1)  # 1==Normal
-        elif dataset_name == 'ctg.2':
-            y = binarize(y, pos_class=2)  # 2==Suspect
-        elif dataset_name == 'ctg.3':
-            y = binarize(y, pos_class=3)  # 3==Pathologic
-
-    if identifier == 'statlog/german':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True)
-        X = df.iloc[:, 0:24].astype(float).values
-        y = df[24].astype(int).values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'haberman':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:3].astype(float).values
-        y = df[3].astype(int).values
-        y = binarize(y, pos_class=2)
-
-    if identifier == 'ionosphere':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:34].astype(float).values
-        y = df[34].values
-        y = binarize(y, pos_class='b')
-
-    if identifier == 'iris':
-        df = pd.read_csv(data_path, header=None)
-        X = df.iloc[:, 0:4].astype(float).values
-        y = df[4].values
-        if dataset_name == 'iris.1':
-            y = binarize(y, pos_class='Iris-setosa')  # 1==Setosa
-        elif dataset_name == 'iris.2':
-            y = binarize(y, pos_class='Iris-versicolor')  # 2==Versicolor
-        elif dataset_name == 'iris.3':
-            y = binarize(y, pos_class='Iris-virginica')  # 3==Virginica
-
-    if identifier == 'mammographic-masses':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        df[df == '?'] = np.nan
-        Xy = df.dropna(axis=0)
-        X = Xy.iloc[:, 0:5]
-        X = X.astype(float).values
-        y = binarize(Xy.iloc[:,5], pos_class=1)
-
-    if identifier == 'page-blocks':
-        data_path_ = data_path.replace('.Z', '')
-        if not os.path.exists(data_path_):
-            raise FileNotFoundError(f'Warning: file {data_path_} does not exist. If this is the first time you '
-                                    f'attempt to load this dataset, then you have to manually unzip the {data_path} '
-                                    f'and name the extracted file {data_path_} (unfortunately, neither zipfile, nor '
-                                    f'gzip can handle unix compressed files automatically -- there is a repo in GitHub '
-                                    f'https://github.com/umeat/unlzw where the problem seems to be solved anyway).')
-        df = pd.read_csv(data_path_, header=None, delim_whitespace=True)
-        X = df.iloc[:, 0:10].astype(float).values
-        y = df[10].values
-        y = binarize(y, pos_class=5)  # 5==block "graphic"
-
-    if identifier == 'semeion':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True )
-        X = df.iloc[:, 0:256].astype(float).values
-        y = df[263].values  # 263 stands for digit 8 (labels are one-hot vectors from col 256-266)
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'undocumented/connectionist-bench/sonar':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:60].astype(float).values
-        y = df[60].values
-        y = binarize(y, pos_class='R')
-
-    if identifier == 'spambase':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:57].astype(float).values
-        y = df[57].values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'spect':
-        dfs = []
-        for file in filename:
-            data_path = join(data_dir, file)
-            download_file_if_not_exists(f'{URL}/{file}', data_path)
-            dfs.append(pd.read_csv(data_path, header=None, sep=','))
-        df = pd.concat(dfs)
-        X = df.iloc[:, 1:45].astype(float).values
-        y = df[0].values
-        y = binarize(y, pos_class=0)
-
-    if identifier == 'tic-tac-toe':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 0:9].replace('o',0).replace('b',1).replace('x',2).values
-        y = df[9].values
-        y = binarize(y, pos_class='negative')
-
-    if identifier == 'blood-transfusion':
-        df = pd.read_csv(data_path, sep=',')
-        X = df.iloc[:, 0:4].astype(float).values
-        y = df.iloc[:, 4].values
-        y = binarize(y, pos_class=1)
-
-    if identifier == 'wine':
-        df = pd.read_csv(data_path, header=None, sep=',')
-        X = df.iloc[:, 1:14].astype(float).values
-        y = df[0].values
-        if dataset_name == 'wine.1':
-            y = binarize(y, pos_class=1)
-        elif dataset_name == 'wine.2':
-            y = binarize(y, pos_class=2)
-        elif dataset_name == 'wine.3':
-            y = binarize(y, pos_class=3)
-
-    if identifier == 'wine-quality':
-        filename = filename[0] if dataset_name=='wine-q-red' else filename[1]
+        :param url_group: identifier of the dataset group in the URL
+        :param filename: name of the file to be downloaded
+        """
+        data_dir = join(data_home, "uci_datasets", "tmp")
+        os.makedirs(data_dir, exist_ok=True)
         data_path = join(data_dir, filename)
-        download_file_if_not_exists(f'{URL}/{filename}', data_path)
-        df = pd.read_csv(data_path, sep=';')
-        X = df.iloc[:, 0:11].astype(float).values
-        y = df.iloc[:, 11].values > 5
+        url = f"http://archive.ics.uci.edu/ml/machine-learning-databases/{url_group}/{filename}"
+        download_file_if_not_exists(url, data_path)
+        try:
+            yield data_path
+        finally:
+            os.remove(data_path)
 
-    if identifier == 'yeast':
-        df = pd.read_csv(data_path, header=None, delim_whitespace=True)
-        X = df.iloc[:, 1:9].astype(float).values
-        y = df.iloc[:, 9].values
-        y = binarize(y, pos_class='NUC')
+    def download(id: int | None, group: str) -> dict:
+        """
+        Download the data to be pickled for a dataset group. Use the `fetch_ucirepo` api when possible.
 
-    data = LabelledCollection(X, y)
+        :param id: numeric identifier for the group; can be None
+        :param group: group name
+        :return: a dictionary with X and y as keys and, optionally, extra data.
+        """
+
+        # use the fetch_ucirepo api, when possible, to download data
+        # fall back to direct download when needed
+        if group == "german":
+            with download_tmp_file("statlog/german", "german.data-numeric") as tmp:
+                df = pd.read_csv(tmp, header=None, delim_whitespace=True)
+            X, y = df.iloc[:, 0:24].astype(float).values, df[24].astype(int).values
+        elif group == "ctg":
+            with download_tmp_file("00193", "CTG.xls") as tmp:
+                df = pd.read_excel(tmp, sheet_name="Data", skipfooter=3)
+            df = df[list(range(1, 24))]  # select columns numbered (number 23 is the target label)
+            # replaces the header with the first row
+            new_header = df.iloc[0]  # grab the first row for the header
+            df = df[1:]  # take the data less the header row
+            df.columns = new_header  # set the header row as the df header
+            X = df.iloc[:, 0:21].astype(float).values  # column 21 is skipped, it is a class column
+            y = df["NSP"].astype(int).values
+        elif group == "semeion":
+            with download_tmp_file("semeion", "semeion.data") as tmp:
+                df = pd.read_csv(tmp, header=None, sep='\\s+')
+            X = df.iloc[:, 0:256].astype(float).values
+            y = df[263].values  # 263 stands for digit 8 (labels are one-hot vectors from col 256-266)
+        else:
+            df = _fetch_ucirepo(id=id)
+            X, y = df.data.features.to_numpy(), df.data.targets.to_numpy().squeeze()
+
+        # transform data when needed before returning (returned data will be pickled)
+        if group == "acute":
+            _array_replace(X)
+            data = {"X": X, "y": y}
+        elif group == "balance":
+            # features' order is reversed to match data retrieved via direct download
+            X = X[:, np.arange(X.shape[1])[::-1]]
+            data = {"X": X, "y": y}
+        elif group == "breast-cancer":
+            # remove rows with nan values
+            Xy = np.hstack([X, y[:, np.newaxis]])
+            nan_rows = np.isnan(Xy).sum(axis=-1) > 0
+            Xy = Xy[~nan_rows]
+            data = {"X": Xy[:, :-1], "y": Xy[:, -1]}
+        elif group == "mammographic":
+            # remove rows with nan values
+            Xy = np.hstack([X, y[:, np.newaxis]])
+            nan_rows = np.isnan(Xy).sum(axis=-1) > 0
+            Xy = Xy[~nan_rows]
+            data = {"X": Xy[:, :-1], "y": Xy[:, -1]}
+        elif group == "tictactoe":
+            _array_replace(X, repl={"o": 0, "b": 1, "x": 2})
+            data = {"X": X, "y": y}
+        elif group == "wine-quality":
+            # add color data to split the final datasets
+            color = df.data.original["color"].to_numpy()
+            data = {"X": X, "y": y, "color": color}
+        else:
+            data = {"X": X, "y": y}
+
+        return data
+
+    def binarize_data(name, data: dict) -> LabelledCollection:
+        """
+        Filter and transform data to extract a binary dataset.
+
+        :param name: name of the dataset
+        :param data: dictionary containing X and y fields, plus additional data when needed
+        :return: a :class:`quapy.data.base.LabelledCollection` with the extracted dataset
+        """
+        if name == "acute.a":
+            X, y = data["X"], data["y"][:, 0]
+        elif name == "acute.b":
+            X, y = data["X"], data["y"][:, 1]
+        elif name == "wine-q-red":
+            X, y, color = data["X"], data["y"], data["color"]
+            red_idx = color == "red"
+            X, y = X[red_idx, :], y[red_idx]
+            y = (y > 5).astype(int)
+        elif name == "wine-q-white":
+            X, y, color = data["X"], data["y"], data["color"]
+            white_idx = color == "white"
+            X, y = X[white_idx, :], y[white_idx]
+            y = (y > 5).astype(int)
+        else:
+            X, y = data["X"], data["y"]
+
+        y = binarize(y, pos_class=pos_class[name])
+
+        return LabelledCollection(X, y)
+
+    data = pickled_resource(file, download, identifier, dataset_group)
+    data = binarize_data(dataset_name, data)
+
+    if standardize:
+        stds = StandardScaler()
+        data.instances = stds.fit_transform(data.instances)
+        
     if verbose:
         data.stats()
+
     return data

 
 
-
[docs]def fetch_UCIMulticlassDataset(dataset_name, data_home=None, test_split=0.3, verbose=False) -> Dataset:
+
+

+[docs]
+def fetch_UCIMulticlassDataset(
+        dataset_name,
+        data_home=None,
+        min_test_split=0.3,
+        max_train_instances=25000,
+        min_class_support=100,
+        standardize=True,
+        verbose=False) -> Dataset:
     """
     Loads a UCI multiclass dataset as an instance of :class:`quapy.data.base.Dataset`. 
 
@@ -651,15 +1011,41 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param min_test_split: minimum proportion of instances to be included in the test set. This value is interpreted
+        as a minimum proportion, meaning that the real proportion could be higher in case the training proportion
+        (1-`min_test_split`% of the instances) surpasses `max_train_instances`. In such case, only `max_train_instances`
+        are taken for training, and the rest (irrespective of `min_test_split`) is taken for test.
+    :param max_train_instances: maximum number of instances to keep for training (defaults to 25000);
+        set to -1 or None to avoid this check
+    :param min_class_support: integer or float, the minimum number or proportion of istances per class.
+        Classes with fewer instances are discarded (deafult is 100).
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,
+        standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the
+        training portion of the data.
     :param verbose: set to True (default is False) to get information (stats) about the dataset
     :return: a :class:`quapy.data.base.Dataset` instance
     """
-    data = fetch_UCIMulticlassLabelledCollection(dataset_name, data_home, verbose)
-    return Dataset(*data.split_stratified(1 - test_split, random_state=0))

+
+    data = fetch_UCIMulticlassLabelledCollection(dataset_name, data_home, min_class_support, verbose=verbose)
+    n = len(data)
+    train_prop = (1.-min_test_split)
+    if (max_train_instances is not None) and (max_train_instances > 0):
+        n_train = int(n*train_prop)
+        if n_train > max_train_instances:
+            train_prop = (max_train_instances / n)
+
+    data = Dataset(*data.split_stratified(train_prop, random_state=0), name=dataset_name)
+    
+    if standardize:
+        data = standardizer(data)
+    
+    return data

 
 
-
[docs]def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, verbose=False) -> LabelledCollection:
+
+

+[docs]
+def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_class_support=100, standardize=True, verbose=False) -> LabelledCollection:
     """
     Loads a UCI multiclass collection as an instance of :class:`quapy.data.base.LabelledCollection`.
 
@@ -681,7 +1067,9 @@
     :param dataset_name: a dataset name
     :param data_home: specify the quapy home directory where the dataset will be dumped (leave empty to use the default
         ~/quay_data/ directory)
-    :param test_split: proportion of documents to be included in the test set. The rest conforms the training set
+    :param min_class_support: minimum number of istances per class. Classes with fewer instances
+        are discarded (deafult is 100)
+    :param standardize: indicates whether the covariates should be standardized or not (default is True). 
     :param verbose: set to True (default is False) to get information (stats) about the dataset
     :return: a :class:`quapy.data.base.LabelledCollection` instance
     """
@@ -689,24 +1077,67 @@
         f'Name {dataset_name} does not match any known dataset from the ' \
         f'UCI Machine Learning datasets repository (multiclass). ' \
         f'Valid ones are {UCI_MULTICLASS_DATASETS}'
+
+    assert (min_class_support is None or
+            ((isinstance(min_class_support, int) and min_class_support >= 0) or
+             (isinstance(min_class_support, float) and 0. <= min_class_support < 1.))), \
+        f'invalid value for {min_class_support=}; expected non negative integer or float in [0,1)'
     
     if data_home is None:
         data_home = get_quapy_home()
     
     identifiers = {
-        "dry-bean": 602,
-        "wine-quality": 186,
-        "academic-success": 697,
-        "digits": 80,
-        "letter": 59
+        'dry-bean': 602,
+        'wine-quality': 186,
+        'academic-success': 697,
+        'digits': 80,
+        'letter': 59,
+        'abalone': 1,
+        'obesity': 544,
+        'nursery': 76,
+        'yeast': 110,
+        'hand_digits': 81,
+        'satellite': 146,
+        'shuttle': 148,
+        'cmc': 30,
+        'isolet': 54,
+        'waveform-v1': 107,
+        'molecular': 69,
+        'poker_hand': 158,
+        'connect-4': 26,
+        'mhr': 863,
+        'chess': 23,
+        'page_block': 78,
+        'phishing': 379,
+        'image_seg': 147,
+        'hcv': 503,
     }
     
     full_names = {
-        "dry-bean": "Dry Bean Dataset",
-        "wine-quality": "Wine Quality",
-        "academic-success": "Predict students' dropout and academic success",
-        "digits": "Optical Recognition of Handwritten Digits",
-        "letter": "Letter Recognition"
+        'dry-bean': 'Dry Bean Dataset',
+        'wine-quality': 'Wine Quality',
+        'academic-success': 'Predict students\' dropout and academic success',
+        'digits': 'Optical Recognition of Handwritten Digits',
+        'letter': 'Letter Recognition',
+        'abalone': 'Abalone',
+        'obesity': 'Estimation of Obesity Levels Based On Eating Habits and Physical Condition',
+        'nursery': 'Nursery',
+        'yeast': 'Yeast',
+        'hand_digits': 'Pen-Based Recognition of Handwritten Digits',
+        'satellite': 'Statlog Landsat Satellite',
+        'shuttle': 'Statlog Shuttle',
+        'cmc': 'Contraceptive Method Choice',
+        'isolet': 'ISOLET',
+        'waveform-v1': 'Waveform Database Generator (Version 1)',
+        'molecular': 'Molecular Biology (Splice-junction Gene Sequences)',
+        'poker_hand': 'Poker Hand',
+        'connect-4': 'Connect-4',
+        'mhr': 'Maternal Health Risk',
+        'chess': 'Chess (King-Rook vs. King)',
+        'page_block': 'Page Blocks Classification',
+        'phishing': 'Website Phishing',
+        'image_seg': 'Statlog (Image Segmentation)',
+        'hcv': 'Hepatitis C Virus (HCV) for Egyptian patients',
     }
     
     identifier = identifiers[dataset_name]
@@ -717,14 +1148,59 @@
 
     file = join(data_home, 'uci_multiclass', dataset_name+'.pkl')
     
-    def download(id):
-        data = fetch_ucirepo(id=id)
-        X, y = data['data']['features'].to_numpy(), data['data']['targets'].to_numpy().squeeze()
+    def dummify_categorical_features(df_features, dataset_id):
+        categorical_features = {
+            158: ["S1", "C1", "S2", "C2", "S3", "C3", "S4", "C4", "S5", "C5"],  # poker_hand
+        }
+
+        categorical = categorical_features.get(dataset_id, [])
+
+        X = df_features.copy()
+        if categorical:
+            X[categorical] = X[categorical].astype("category")
+            X = pd.get_dummies(X, columns=categorical, drop_first=True)
+
+        return X
+
+    def download(id, name):
+        df = _fetch_ucirepo(id=id)
+
+        X_df = dummify_categorical_features(df.data.features, id)
+        X = X_df.to_numpy(dtype=np.float64)
+        y = df.data.targets.to_numpy().squeeze()
+
+        assert y.ndim == 1, f'error: the dataset {id=} {name=} has more than one target variable'
+
         classes = np.sort(np.unique(y))
         y = np.searchsorted(classes, y)
         return LabelledCollection(X, y)
 
-    data = pickled_resource(file, download, identifier)
+    def filter_classes(data: LabelledCollection, min_class_support):
+        if min_class_support is None or min_class_support == 0.:
+            return data
+        if isinstance(min_class_support, float):
+            min_class_support = int(len(data) * min_class_support)
+        classes = data.classes_
+        # restrict classes to only those with at least min_class_support instances
+        classes = classes[data.counts() >= min_class_support]
+        # filter X and y keeping only datapoints belonging to valid classes
+        filter_idx = np.isin(data.y, classes)
+        X, y = data.X[filter_idx], data.y[filter_idx]
+        # map classes to range(len(classes))
+        y = np.searchsorted(classes, y)
+        return LabelledCollection(X, y)
+
+    data = pickled_resource(file, download, identifier, dataset_name)
+    data = filter_classes(data, min_class_support)
+    if data.n_classes <= 2:
+        raise ValueError(
+            f'After filtering out classes with less than {min_class_support=} instances, the dataset {dataset_name} '
+            f'is no longer multiclass. Try a reducing this value.'
+        )
+
+    if standardize:
+        stds = StandardScaler()
+        data.instances = stds.fit_transform(data.instances)
 
     if verbose:
         data.stats()
@@ -732,13 +1208,21 @@
     return data

 
 
-def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float):
+
+def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float):
     df[col] = df[col].apply(lambda x:repl[x]).astype(astype, copy=False)
 
 
-
[docs]def fetch_lequa2022(task, data_home=None):
+def _array_replace(arr, repl={"yes": 1, "no": 0}):
+    for k, v in repl.items():
+        arr[arr == k] = v
+
+
+

+[docs]
+def fetch_lequa2022(task, data_home=None):
     """
-    Loads the official datasets provided for the `LeQua <https://lequa2022.github.io/index>`_ competition.
+    Loads the official datasets provided for the `LeQua 2022 <https://lequa2022.github.io/index>`_ competition.
     In brief, there are 4 tasks (T1A, T1B, T2A, T2B) having to do with text quantification
     problems. Tasks T1A and T1B provide documents in vector form, while T2A and T2B provide raw documents instead.
     Tasks T1A and T2A are binary sentiment quantification problems, while T2A and T2B are multiclass quantification
@@ -750,20 +1234,19 @@
 
     The datasets are downloaded only once, and stored for fast reuse.
 
-    See `lequa2022_experiments.py` provided in the example folder, that can serve as a guide on how to use these
+    See `4.lequa2022_experiments.py` provided in the example folder, that can serve as a guide on how to use these
     datasets.
 
-
     :param task: a string representing the task name; valid ones are T1A, T1B, T2A, and T2B
     :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
         ~/quay_data/ directory)
     :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of
         :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of
-        :class:`quapy.data._lequa2022.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
+        :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
         that return a series of samples stored in a directory which are labelled by prevalence.
     """
 
-    from quapy.data._lequa2022 import load_raw_documents, load_vector_documents, SamplesFromDir
+    from quapy.data._lequa import load_raw_documents, load_vector_documents_2022, SamplesFromDir
 
     assert task in LEQUA2022_TASKS, \
         f'Unknown task {task}. Valid ones are {LEQUA2022_TASKS}'
@@ -777,11 +1260,13 @@
     lequa_dir = join(data_home, 'lequa2022')
     os.makedirs(lequa_dir, exist_ok=True)
 
-    def download_unzip_and_remove(unzipped_path, url):
+    def download_unzip_and_remove(unzipped_path, url):
         tmp_path = join(lequa_dir, task + '_tmp.zip')
         download_file_if_not_exists(url, tmp_path)
         with zipfile.ZipFile(tmp_path) as file:
+            print(f'Unzipping {tmp_path}...', end='')
             file.extractall(unzipped_path)
+            print(f'[done]')
         os.remove(tmp_path)
 
     if not os.path.exists(join(lequa_dir, task)):
@@ -790,7 +1275,7 @@
         download_unzip_and_remove(lequa_dir, URL_TEST_PREV)
 
     if task in ['T1A', 'T1B']:
-        load_fn = load_vector_documents
+        load_fn = load_vector_documents_2022
     elif task in ['T2A', 'T2B']:
         load_fn = load_raw_documents
 
@@ -808,14 +1293,107 @@
     return train, val_gen, test_gen

 
 
-
[docs]def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None):
+
+

+[docs]
+def fetch_lequa2024(task, data_home=None, merge_T3=False):
+    """
+    Loads the official datasets provided for the `LeQua 2024 <https://lequa2024.github.io/index>`_ competition.
+    LeQua 2024 defines four tasks (T1, T2, T3, T4) related to the problem of quantification;
+    all tasks are affected by some type of dataset shift. Tasks T1 and T2 are akin to tasks T1A and T1B of LeQua 2022,
+    while T3 and T4 are new tasks introduced in LeQua 2024.
+
+    - Task T1 evaluates binary quantifiers under prior probability shift (akin to T1A of LeQua 2022).
+    - Task T2 evaluates single-label multi-class quantifiers (for n > 2 classes) under prior probability shift (akin to T1B of LeQua 2022).
+    - Task T3 evaluates ordinal quantifiers, where the classes are totally ordered.
+    - Task T4 also evaluates binary quantifiers, but under some mix of covariate shift and prior probability shift.
+
+    For a broader discussion, we refer to the `online official documentation <https://lequa2024.github.io/tasks/>`_
+
+    The datasets are downloaded only once, and stored locally for future reuse.
+
+    See `4b.lequa2024_experiments.py` provided in the example folder, which can serve as a guide on how to use these
+    datasets.
+
+    :param task: a string representing the task name; valid ones are T1, T2, T3, and T4
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quapy_data/ directory)
+    :param merge_T3: bool, if False (default), returns a generator of training collections, corresponding to natural
+        groups of reviews; if True, returns one single :class:`quapy.data.base.LabelledCollection` representing the
+        entire training set, as a concatenation of all the training collections
+    :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of
+        :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of
+        :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,
+        that return a series of samples stored in a directory which are labelled by prevalence.
+    """
+
+    from quapy.data._lequa import load_vector_documents_2024, SamplesFromDir, LabelledCollectionsFromDir
+
+    assert task in LEQUA2024_TASKS, \
+        f'Unknown task {task}. Valid ones are {LEQUA2024_TASKS}'
+
+    if data_home is None:
+        data_home = get_quapy_home()
+
+    lequa_dir = data_home
+
+    LEQUA2024_ZENODO = 'https://zenodo.org/records/11661820'  # v3, last one with labels
+
+    URL_TRAINDEV=f'{LEQUA2024_ZENODO}/files/{task}.train_dev.zip'
+    URL_TEST=f'{LEQUA2024_ZENODO}/files/{task}.test.zip'
+    URL_TEST_PREV=f'{LEQUA2024_ZENODO}/files/{task}.test_prevalences.zip'
+
+    lequa_dir = join(data_home, 'lequa2024')
+    os.makedirs(lequa_dir, exist_ok=True)
+
+    def download_unzip_and_remove(unzipped_path, url):
+        tmp_path = join(lequa_dir, task + '_tmp.zip')
+        download_file_if_not_exists(url, tmp_path)
+        with zipfile.ZipFile(tmp_path) as file:
+            file.extractall(unzipped_path)
+        os.remove(tmp_path)
+
+    if not os.path.exists(join(lequa_dir, task)):
+        download_unzip_and_remove(lequa_dir, URL_TRAINDEV)
+        download_unzip_and_remove(lequa_dir, URL_TEST)
+        download_unzip_and_remove(lequa_dir, URL_TEST_PREV)
+
+    load_fn = load_vector_documents_2024
+
+    val_samples_path = join(lequa_dir, task, 'public', 'dev_samples')
+    val_true_prev_path = join(lequa_dir, task, 'public', 'dev_prevalences.txt')
+    val_gen = SamplesFromDir(val_samples_path, val_true_prev_path, load_fn=load_fn)
+
+    test_samples_path = join(lequa_dir, task, 'public', 'test_samples')
+    test_true_prev_path = join(lequa_dir, task, 'public', 'test_prevalences.txt')
+    test_gen = SamplesFromDir(test_samples_path, test_true_prev_path, load_fn=load_fn)
+
+    if task == 'T3':
+        training_samples_path = join(lequa_dir, task, 'public', 'training_samples')
+        training_true_prev_path = join(lequa_dir, task, 'public', 'training_prevalences.txt')
+        train_gen = LabelledCollectionsFromDir(training_samples_path, training_true_prev_path, load_fn=load_fn)
+        if merge_T3:
+            train = LabelledCollection.join(*list(train_gen()))
+            return train, val_gen, test_gen
+        else:
+            return train_gen, val_gen, test_gen
+    else:
+        tr_path = join(lequa_dir, task, 'public', 'training_data.txt')
+        train = LabelledCollection.load(tr_path, loader_func=load_fn)
+        return train, val_gen, test_gen

+
+
+
+

+[docs]
+def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None):
     """
     Loads the IFCB dataset for quantification from `Zenodo <https://zenodo.org/records/10036244>`_ (for more
     information on this dataset, please follow the zenodo link).
     This dataset is based on the data available publicly at
     `WHOI-Plankton repo <https://github.com/hsosik/WHOI-Plankton>`_.
-    The scripts for the processing are available at `P. González's repo <https://github.com/pglez82/IFCB_Zenodo>`_.
-    Basically, this is the IFCB dataset with precomputed features for testing quantification algorithms.
+    The dataset already comes with processed features.
+    The scripts used for the processing are available at `P. González's repo <https://github.com/pglez82/IFCB_Zenodo>`_.
 
     The datasets are downloaded only once, and stored for fast reuse.
 
@@ -833,7 +1411,7 @@
         i.e., a sampling protocol that returns a series of samples labelled by prevalence.
     """
 
-    from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split
+    from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split
 
     if data_home is None:
         data_home = get_quapy_home()
@@ -845,7 +1423,7 @@
     ifcb_dir = join(data_home, 'ifcb')
     os.makedirs(ifcb_dir, exist_ok=True)
 
-    def download_unzip_and_remove(unzipped_path, url):
+    def download_unzip_and_remove(unzipped_path, url):
         tmp_path = join(ifcb_dir, 'ifcb_tmp.zip')
         download_file_if_not_exists(url, tmp_path)
         with zipfile.ZipFile(tmp_path) as file:
@@ -871,7 +1449,7 @@
     if for_model_selection:
         # In this case, return 70% of training data as the training set and 30% as the test set
         samples = get_sample_list(train_samples_path)
-        train, test = generate_modelselection_split(samples, split=0.3)
+        train, test = generate_modelselection_split(samples, test_prop=0.3)
         train_gen = IFCBTrainSamplesFromDir(path_dir=train_samples_path, classes=classes, samples=train)
 
         # Test prevalence is computed from class labels
@@ -887,33 +1465,168 @@
         return train, test_gen
     else:
         return train_gen, test_gen

+
+
+
+def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]:
+    """
+    Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo <https://zenodo.org/records/21131944>`_.
+    
+    Embeddings were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. 
+
+    :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'
+    :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation values), 'predictions' (posterior probabilities)
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quay_data/ directory)
+    :return: a tuple (train, val, test) where each entry is an instance of :class:`quapy.data.base.LabelledCollection`
+    """
+    assert dataset_name in IMAGE_DATASETS, \
+        f'Name {dataset_name} does not match any known dataset. Valid ones are {IMAGE_DATASETS}'
+    assert embedding in IMAGE_EMBEDDINGS, \
+        f'Name {embedding} does not match any known type of embedding. Valid ones are {IMAGE_EMBEDDINGS}'
+    if data_home is None:
+        data_home = get_quapy_home()
+    
+    dataset_network = {
+        'cifar10': 'resnet18',
+        'cifar100': 'resnet18',
+        'cifar100coarse': 'resnet18',
+        'svhn': 'resnet18',
+        'fashionmnist': 'basiccnn',
+        'mnist': 'basiccnn',
+    }
+
+    trained_network = dataset_network[dataset_name]
+
+    def download_embedding_npz(dataset_name, trained_network, embedding):
+        target_file = f'{dataset_name}_{trained_network}_{embedding}.npz'
+        URL = f'https://zenodo.org/records/21131944/files/{target_file}'
+        os.makedirs(join(data_home, 'image'), exist_ok=True)
+        file_path = join(data_home, 'image', target_file)
+        download_file_if_not_exists(URL, file_path)
+        npz_file = np.load(file_path)
+        return npz_file
+
+    embedding_dict = download_embedding_npz(dataset_name, trained_network, embedding=embedding)
+    labels_dict = download_embedding_npz(dataset_name, trained_network, embedding='targets')
+
+    train = LabelledCollection(embedding_dict['train'], labels_dict['train'])
+    val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes)
+    test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes)
+
+    return train, val, test
+
+
+

+[docs]
+def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset:
+    """
+    Loads an image dataset with pre-generated embeddings. Available datasets include:
+
+    - 'cifar10', 'cifar100', 'cifar100coarse': see `Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, Toronto, Ontario, 2009. <https://cave.cs.toronto.edu/kriz/learning-features-2009-TR.pdf>`_
+    - 'mnist': `Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998. <http://yann.lecun.com/exdb/mnist/>`_
+    - 'fashionmnist': `Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017. <https://arxiv.org/abs/1708.07747>`_
+    - 'svhn': `Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y Ng, et al. Reading digits in natural images with unsupervised feature learning. In NIPS workshop on deep learning and unsupervised feature learning, volume 2011, page 4. Granada, 2011. <https://static.googleusercontent.com/media/research.google.com/es//pubs/archive/37648.pdf>`_
+    
+    The image dataset are stored in `Zenodo <https://zenodo.org/records/21131944>`_ and were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. 
+
+    These embeddings were generated using a resnet18 or a simple cnn. In all cases, the network was trained using ~60% of the data, validated on ~25% of the data, and the remaining ~15% was used for test. Splits were created with stratification.
+    Once the network is trained, it was used with frozen weights to generate embeddings for the training, validation, and test, in different formats (see below).
+    It would therefore be convenient to use only heldout data (validation and test) for training and testing quantifiers (this is the default behavior), although the training+validation data can be accessed with `heldout_only=False`.
+
+    :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'
+    :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation outputs), 'predictions' (post-softmax outputs, or predicted posterior probabilities)
+    :param heldout_only: whether to discard the part of the training data used to train the neural model that generated the embeddings (default: True); set to False
+        to obtain, as the training data, the original training+validation splits.
+    :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default
+        ~/quay_data/ directory)
+    :return: an instance of :class:`quapy.data.base.Dataset`
+    """
+    if data_home is None:
+        data_home = get_quapy_home()
+
+    network_train, val, test = _fetch_image_embedding_splits(dataset_name, embedding, data_home)        
+
+    if heldout_only:
+        train = val            
+    else:
+        train = network_train + val
+        
+    return Dataset(train, test, name=dataset_name)

+
+
+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/data/preprocessing.html b/docs/build/html/_modules/quapy/data/preprocessing.html index a50aa64..dcfe434 100644 --- a/docs/build/html/_modules/quapy/data/preprocessing.html +++ b/docs/build/html/_modules/quapy/data/preprocessing.html @@ -1,22 +1,20 @@ + + - quapy.data.preprocessing — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.data.preprocessing — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - + + + + + @@ -42,7 +40,13 @@ @@ -70,21 +74,55 @@

Source code for quapy.data.preprocessing

-import numpy as np
-from scipy.sparse import spmatrix
-from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
-from sklearn.preprocessing import StandardScaler
-from tqdm import tqdm
+import numpy as np
+from scipy.sparse import spmatrix
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+
+import quapy as qp
+from quapy.data.base import Dataset
+from quapy.util import map_parallel
+from .base import LabelledCollection
+
+
+

+[docs]
+def instance_transformation(dataset:Dataset, transformer, inplace=False):
+    """
+    Transforms a :class:`quapy.data.base.Dataset` applying the `fit_transform` and `transform` functions
+    of a (sklearn's) transformer.
+
+    :param dataset: a :class:`quapy.data.base.Dataset` where the instances of training and test collections are
+        lists of str
+    :param transformer: TransformerMixin implementing `fit_transform` and `transform` functions
+    :param inplace: whether or not to apply the transformation inplace (True), or to a new copy (False, default)
+    :return: a new :class:`quapy.data.base.Dataset` with transformed instances (if inplace=False) or a reference to the
+        current Dataset (if inplace=True) where the instances have been transformed
+    """
+    training_transformed = transformer.fit_transform(*dataset.training.Xy)
+    test_transformed = transformer.transform(dataset.test.X)
+    orig_name = dataset.name
+
+    if inplace:
+        dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_)
+        dataset.test = LabelledCollection(test_transformed, dataset.test.labels, dataset.classes_)
+        if hasattr(transformer, 'vocabulary_'):
+            dataset.vocabulary = transformer.vocabulary_
+        return dataset
+    else:
+        training = LabelledCollection(training_transformed, dataset.training.labels.copy(), dataset.classes_)
+        test = LabelledCollection(test_transformed, dataset.test.labels.copy(), dataset.classes_)
+        vocab = None
+        if hasattr(transformer, 'vocabulary_'):
+            vocab = transformer.vocabulary_
+        return Dataset(training, test, vocabulary=vocab, name=orig_name)

 
-import quapy as qp
-from quapy.data.base import Dataset
-from quapy.util import map_parallel
-from .base import LabelledCollection
 
 
 

 [docs]
-def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs):
+def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs):
     """
     Transforms a :class:`quapy.data.base.Dataset` of textual instances into a :class:`quapy.data.base.Dataset` of
     tfidf weighted sparse vectors
@@ -103,24 +141,13 @@
     __check_type(dataset.test.instances, np.ndarray, str)
 
     vectorizer = TfidfVectorizer(min_df=min_df, sublinear_tf=sublinear_tf, **kwargs)
-    training_documents = vectorizer.fit_transform(dataset.training.instances)
-    test_documents = vectorizer.transform(dataset.test.instances)
-
-    if inplace:
-        dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.classes_)
-        dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.classes_)
-        dataset.vocabulary = vectorizer.vocabulary_
-        return dataset
-    else:
-        training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.classes_)
-        test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.classes_)
-        return Dataset(training, test, vectorizer.vocabulary_)

+    return instance_transformation(dataset, vectorizer, inplace)
[docs] -def reduce_columns(dataset: Dataset, min_df=5, inplace=False): +def reduce_columns(dataset: Dataset, min_df=5, inplace=False): """ Reduces the dimensionality of the instances, represented as a `csr_matrix` (or any subtype of `scipy.sparse.spmatrix`), of training and test documents by removing the columns of words which are not present @@ -138,7 +165,7 @@ __check_type(dataset.test.instances, spmatrix) assert dataset.training.instances.shape[1] == dataset.test.instances.shape[1], 'unaligned vector spaces' - def filter_by_occurrences(X, W): + def filter_by_occurrences(X, W): column_prevalence = np.asarray((X > 0).sum(axis=0)).flatten() take_columns = column_prevalence >= min_df X = X[:, take_columns] @@ -159,7 +186,7 @@
[docs] -def standardize(dataset: Dataset, inplace=False): +def standardize(dataset: Dataset, inplace=False): """ Standardizes the real-valued columns of a :class:`quapy.data.base.Dataset`. Standardization, aka z-scoring, of a variable `X` comes down to subtracting the average and normalizing by the @@ -170,19 +197,24 @@ :class:`quapy.data.base.Dataset` is to be returned :return: an instance of :class:`quapy.data.base.Dataset` """ - s = StandardScaler(copy=not inplace) - training = s.fit_transform(dataset.training.instances) - test = s.transform(dataset.test.instances) + s = StandardScaler() + train, test = dataset.train_test + std_train_X = s.fit_transform(train.X) + std_test_X = s.transform(test.X) if inplace: + dataset.training.instances = std_train_X + dataset.test.instances = std_test_X return dataset else: + training = LabelledCollection(std_train_X, train.labels, classes=train.classes_) + test = LabelledCollection(std_test_X, test.labels, classes=test.classes_) return Dataset(training, test, dataset.vocabulary, dataset.name)
[docs] -def index(dataset: Dataset, min_df=5, inplace=False, **kwargs): +def index(dataset: Dataset, min_df=5, inplace=False, **kwargs): """ Indexes the tokens of a textual :class:`quapy.data.base.Dataset` of string documents. To index a document means to replace each different token by a unique numerical index. @@ -219,7 +251,7 @@ -def __check_type(container, container_type=None, element_type=None): +def __check_type(container, container_type=None, element_type=None): if container_type: assert isinstance(container, container_type), \ f'unexpected type of container (expected {container_type}, found {type(container)})' @@ -230,7 +262,7 @@
[docs] -class IndexTransformer: +class IndexTransformer: """ This class implements a sklearn's-style transformer that indexes text as numerical ids for the tokens it contains, and that would be generated by sklearn's @@ -240,14 +272,14 @@ `CountVectorizer <https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html>`_ """ - def __init__(self, **kwargs): + def __init__(self, **kwargs): self.vect = CountVectorizer(**kwargs) self.unk = -1 # a valid index is assigned after fit self.pad = -2 # a valid index is assigned after fit
[docs] - def fit(self, X): + def fit(self, X): """ Fits the transformer, i.e., decides on the vocabulary, given a list of strings. @@ -264,7 +296,7 @@
[docs] - def transform(self, X, n_jobs=None): + def transform(self, X, n_jobs=None): """ Transforms the strings in `X` as lists of numerical ids @@ -279,13 +311,13 @@ - def _index(self, documents): + def _index(self, documents): vocab = self.vocabulary_.copy() return [[vocab.get(word, self.unk) for word in self.analyzer(doc)] for doc in tqdm(documents, 'indexing')]
[docs] - def fit_transform(self, X, n_jobs=None): + def fit_transform(self, X, n_jobs=None): """ Fits the transform on `X` and transforms it. @@ -298,7 +330,7 @@
[docs] - def vocabulary_size(self): + def vocabulary_size(self): """ Gets the length of the vocabulary according to which the document tokens have been indexed @@ -309,7 +341,7 @@
[docs] - def add_word(self, word, id=None, nogaps=True): + def add_word(self, word, id=None, nogaps=True): """ Adds a new token (regardless of whether it has been found in the text or not), with dedicated id. Useful to define special tokens for codifying unknown words, or padding tokens. diff --git a/docs/build/html/_modules/quapy/data/reader.html b/docs/build/html/_modules/quapy/data/reader.html index 4c9c163..96b2d15 100644 --- a/docs/build/html/_modules/quapy/data/reader.html +++ b/docs/build/html/_modules/quapy/data/reader.html @@ -1,83 +1,385 @@ + - - - - - quapy.data.reader — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.data.reader — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.data.reader

-import numpy as np
-from scipy.sparse import dok_matrix
-from tqdm import tqdm
+import logging
+
+import numpy as np
+from scipy.sparse import dok_matrix
+from tqdm import tqdm
 
 
 

 [docs]
-def from_text(path, encoding='utf-8', verbose=1, class2int=True):
+def from_text(path, encoding='utf-8', verbose=1, class2int=True):
     """
     Reads a labelled colletion of documents.
     File fomart <0 or 1>\t<document>\n
@@ -104,14 +406,14 @@
                     all_sentences.append(sentence)
                     all_labels.append(label)
             except ValueError:
-                print(f'format error in {line}')
+                logging.getLogger(__name__).warning(f'format error in {line}')
     return all_sentences, all_labels

 
 
 
 

 [docs]
-def from_sparse(path):
+def from_sparse(path):
     """
     Reads a labelled collection of real-valued instances expressed in sparse format
     File format <-1 or 0 or 1>[\s col(int):val(float)]\n
@@ -120,7 +422,7 @@
     :return: a `csr_matrix` containing the instances (rows), and a ndarray containing the labels
     """
 
-    def split_col_val(col_val):
+    def split_col_val(col_val):
         col, val = col_val.split(':')
         col, val = int(col) - 1, float(val)
         return col, val
@@ -148,7 +450,7 @@
 
 

 [docs]
-def from_csv(path, encoding='utf-8'):
+def from_csv(path, encoding='utf-8'):
     """
     Reads a csv file in which columns are separated by ','.
     File format <label>,<feat1>,<feat2>,...,<featn>\n
@@ -171,7 +473,7 @@
 
 

 [docs]
-def reindex_labels(y):
+def reindex_labels(y):
     """
     Re-indexes a list of labels as a list of indexes, and returns the classnames corresponding to the indexes.
     E.g.:
@@ -194,7 +496,7 @@
 
 

 [docs]
-def binarize(y, pos_class):
+def binarize(y, pos_class):
     """
     Binarizes a categorical array-like collection of labels towards the positive class `pos_class`. E.g.,:
 
@@ -214,31 +516,75 @@
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
+
+ +
+ +
-
-
-
- -
- +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+
+ +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ +
+ + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ +
+ + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/error.html b/docs/build/html/_modules/quapy/error.html index 1613468..500dc59 100644 --- a/docs/build/html/_modules/quapy/error.html +++ b/docs/build/html/_modules/quapy/error.html @@ -1,84 +1,385 @@ + - - - - - quapy.error — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.error — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.error

 """Implementation of error measures used for quantification"""
 
-import numpy as np
-from sklearn.metrics import f1_score
-import quapy as qp
+import numpy as np
+from sklearn.metrics import f1_score
+import quapy as qp
 
 
-
[docs]def from_name(err_name):
+

+[docs]
+def from_name(err_name):
     """Gets an error function from its name. E.g., `from_name("mae")`
     will return function :meth:`quapy.error.mae`
 
@@ -90,7 +391,10 @@
     return callable_error

 
 
-
[docs]def f1e(y_true, y_pred):
+
+

+[docs]
+def f1e(y_true, y_pred):
     """F1 error: simply computes the error in terms of macro :math:`F_1`, i.e.,
     :math:`1-F_1^M`, where :math:`F_1` is the harmonic mean of precision and recall,
     defined as :math:`\\frac{2tp}{2tp+fp+fn}`, with `tp`, `fp`, and `fn` standing
@@ -105,7 +409,10 @@
     return 1. - f1_score(y_true, y_pred, average='macro')

 
 
-
[docs]def acce(y_true, y_pred):
+
+

+[docs]
+def acce(y_true, y_pred):
     """Computes the error in terms of 1-accuracy. The accuracy is computed as
     :math:`\\frac{tp+tn}{tp+fp+fn+tn}`, with `tp`, `fp`, `fn`, and `tn` standing
     for true positives, false positives, false negatives, and true negatives,
@@ -118,89 +425,207 @@
     return 1. - (y_true == y_pred).mean()

 
 
-
[docs]def mae(prevs, prevs_hat):
+
+

+[docs]
+def mae(prevs_true, prevs_hat):
     """Computes the mean absolute error (see :meth:`quapy.error.ae`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :return: mean absolute error
     """
-    return ae(prevs, prevs_hat).mean()

+    return ae(prevs_true, prevs_hat).mean()

 
 
-
[docs]def ae(prevs, prevs_hat):
+
+

+[docs]
+def ae(prevs_true, prevs_hat):
     """Computes the absolute error between the two prevalence vectors.
      Absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`AE(p,\\hat{p})=\\frac{1}{|\\mathcal{Y}|}\\sum_{y\\in \\mathcal{Y}}|\\hat{p}(y)-p(y)|`,
      where :math:`\\mathcal{Y}` are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: absolute error
     """
-    assert prevs.shape == prevs_hat.shape, f'wrong shape {prevs.shape} vs. {prevs_hat.shape}'
-    return abs(prevs_hat - prevs).mean(axis=-1)

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape} vs. {prevs_hat.shape}'
+    return abs(prevs_hat - prevs_true).mean(axis=-1)

 
 
-
[docs]def nae(prevs, prevs_hat):
+
+

+[docs]
+def nae(prevs_true, prevs_hat):
     """Computes the normalized absolute error between the two prevalence vectors.
      Normalized absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`NAE(p,\\hat{p})=\\frac{AE(p,\\hat{p})}{z_{AE}}`,
      where :math:`z_{AE}=\\frac{2(1-\\min_{y\\in \\mathcal{Y}} p(y))}{|\\mathcal{Y}|}`, and :math:`\\mathcal{Y}`
      are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: normalized absolute error
     """
-    assert prevs.shape == prevs_hat.shape, f'wrong shape {prevs.shape} vs. {prevs_hat.shape}'
-    return abs(prevs_hat - prevs).sum(axis=-1)/(2*(1-prevs.min(axis=-1)))

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape} vs. {prevs_hat.shape}'
+    return abs(prevs_hat - prevs_true).sum(axis=-1)/(2 * (1 - prevs_true.min(axis=-1)))

 
 
-
[docs]def mnae(prevs, prevs_hat):
+
+

+[docs]
+def mnae(prevs_true, prevs_hat):
     """Computes the mean normalized absolute error (see :meth:`quapy.error.nae`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :return: mean normalized absolute error
     """
-    return nae(prevs, prevs_hat).mean()

+    return nae(prevs_true, prevs_hat).mean()

 
 
-
[docs]def mse(prevs, prevs_hat):
+
+

+[docs]
+def mse(prevs_true, prevs_hat):
     """Computes the mean squared error (see :meth:`quapy.error.se`) across the sample pairs.
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the
         true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the
         predicted prevalence values
     :return: mean squared error
     """
-    return se(prevs, prevs_hat).mean()

+    return se(prevs_true, prevs_hat).mean()

 
 
-
[docs]def se(prevs, prevs_hat):
+
+

+[docs]
+def se(prevs_true, prevs_hat):
     """Computes the squared error between the two prevalence vectors.
      Squared error between two prevalence vectors :math:`p` and :math:`\\hat{p}`  is computed as
      :math:`SE(p,\\hat{p})=\\frac{1}{|\\mathcal{Y}|}\\sum_{y\\in \\mathcal{Y}}(\\hat{p}(y)-p(y))^2`,
      where
      :math:`\\mathcal{Y}` are the classes of interest.
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :return: absolute error
     """
-    return ((prevs_hat - prevs) ** 2).mean(axis=-1)

+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    return ((prevs_hat - prevs_true) ** 2).mean(axis=-1)

 
 
-
[docs]def mkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def sre(prevs_true, prevs_hat, prevs_train, eps=0.):
+    """
+    Computes the squared ratio error between two prevalence vectors.
+    The squared ratio error between prevalence vectors :math:`p` and
+    :math:`\\hat{p}` with training prevalence :math:`p^{tr}` is:
+    :math:`SRE(p,\\hat{p},p^{tr})=\\frac{1}{|\\mathcal{Y}|}\\sum_{i \\in \\mathcal{Y}}(w_i-\\hat{w}_i)^2`,
+    where :math:`w_i=\\frac{p_i}{p^{tr}_i}`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :param prevs_train: array-like with the training prevalence values, or a single
+        prevalence vector when all comparisons refer to the same training set
+    :param eps: smoothing factor for the prevalence values (default 0, i.e., no smoothing)
+    :return: squared ratio error
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    prevs_train = np.asarray(prevs_train)
+    assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape=} vs {prevs_hat.shape=}'
+    assert prevs_true.shape[-1] == prevs_train.shape[-1], 'wrong shape for training prevalence'
+    if prevs_true.ndim == 2 and prevs_train.ndim == 1:
+        prevs_train = np.tile(prevs_train, reps=(prevs_true.shape[0], 1))
+    if eps > 0:
+        prevs_true = smooth(prevs_true, eps)
+        prevs_hat = smooth(prevs_hat, eps)
+        prevs_train = smooth(prevs_train, eps)
+
+    n_classes = prevs_true.shape[-1]
+    w = prevs_true / prevs_train
+    w_hat = prevs_hat / prevs_train
+    return (1. / n_classes) * np.sum((w - w_hat) ** 2., axis=-1)

+
+
+
+

+[docs]
+def msre(prevs_true, prevs_hat, prevs_train, eps=0.):
+    """
+    Computes the mean squared ratio error (see :meth:`quapy.error.sre`) across the sample pairs.
+
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape equal to prevs_true with the predicted prevalence values
+    :param prevs_train: array-like with the training prevalence values
+    :param eps: smoothing factor (default 0, i.e., no smoothing)
+    :return: mean squared ratio error
+    """
+    return np.mean(sre(prevs_true, prevs_hat, prevs_train, eps))

+
+
+
+

+[docs]
+def aitchisondist(prevs_true, prevs_hat):
+    """
+    Computes the Aitchison distance between two prevalence vectors.
+    The Aitchison distance between prevalence vectors :math:`p` and
+    :math:`\\hat{p}` is computed as
+    :math:`d_A(p,\\hat{p})=\\|\\mathrm{clr}(p)-\\mathrm{clr}(\\hat{p})\\|_2`,
+    where :math:`\\mathrm{clr}(p)_i=\\log p_i-\\frac{1}{|\\mathcal{Y}|}
+    \\sum_{j \\in \\mathcal{Y}} \\log p_j`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :return: Aitchison distance
+    """
+    from quapy.functional import CLRtransformation
+
+    clr = CLRtransformation()
+    return np.linalg.norm(clr(prevs_true) - clr(prevs_hat), axis=-1)

+
+
+
+

+[docs]
+def maitchisondist(prevs_true, prevs_hat):
+    """
+    Computes the mean Aitchison distance (see :meth:`quapy.error.aitchisondist`)
+    across the sample pairs, i.e.,
+    :math:`\\mathrm{mAitchisonDist}=\\frac{1}{n}\\sum_{i=1}^n
+    d_A(p_i,\\hat{p}_i)`.
+
+    :param prevs_true: array-like with the true prevalence values
+    :param prevs_hat: array-like with the predicted prevalence values
+    :return: mean Aitchison distance
+    """
+    return np.mean(aitchisondist(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def mkld(prevs_true, prevs_hat, eps=None):
     """Computes the mean Kullback-Leibler divergence (see :meth:`quapy.error.kld`) across the
     sample pairs. The distributions are smoothed using the `eps` factor
     (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -210,10 +635,13 @@
         (which has thus to be set beforehand).
     :return: mean Kullback-Leibler distribution
     """
-    return kld(prevs, prevs_hat, eps).mean()

+    return kld(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def kld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def kld(prevs_true, prevs_hat, eps=None):
     """Computes the Kullback-Leibler divergence between the two prevalence distributions.
      Kullback-Leibler divergence between two prevalence distributions :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -222,7 +650,7 @@
      where :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. KLD is not defined in cases in which the distributions contain
         zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the sample size.
@@ -231,17 +659,20 @@
     :return: Kullback-Leibler divergence between the two distributions
     """
     eps = __check_eps(eps)
-    smooth_prevs = prevs + eps
-    smooth_prevs_hat = prevs_hat + eps
+    smooth_prevs = smooth(prevs_true, eps)
+    smooth_prevs_hat = smooth(prevs_hat, eps)
     return (smooth_prevs*np.log(smooth_prevs/smooth_prevs_hat)).sum(axis=-1)

 
 
-
[docs]def mnkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mnkld(prevs_true, prevs_hat, eps=None):
     """Computes the mean Normalized Kullback-Leibler divergence (see :meth:`quapy.error.nkld`)
     across the sample pairs. The distributions are smoothed using the `eps` factor
     (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
     :param eps: smoothing factor. NKLD is not defined in cases in which the distributions contain
@@ -250,10 +681,13 @@
         (which has thus to be set beforehand).
     :return: mean Normalized Kullback-Leibler distribution
     """
-    return nkld(prevs, prevs_hat, eps).mean()

+    return nkld(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def nkld(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def nkld(prevs_true, prevs_hat, eps=None):
     """Computes the Normalized Kullback-Leibler divergence between the two prevalence distributions.
      Normalized Kullback-Leibler divergence between two prevalence distributions :math:`p` and
      :math:`\\hat{p}` is computed as
@@ -262,7 +696,7 @@
      :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. NKLD is not defined in cases in which the distributions
         contain zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the sample
@@ -270,16 +704,19 @@
         `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: Normalized Kullback-Leibler divergence between the two distributions
     """
-    ekld = np.exp(kld(prevs, prevs_hat, eps))
+    ekld = np.exp(kld(prevs_true, prevs_hat, eps))
     return 2. * ekld / (1 + ekld) - 1.

 
 
-
[docs]def mrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mrae(prevs_true, prevs_hat, eps=None):
     """Computes the mean relative absolute error (see :meth:`quapy.error.rae`) across
     the sample pairs. The distributions are smoothed using the `eps` factor (see
     :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -289,10 +726,13 @@
         the environment variable `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: mean relative absolute error
     """
-    return rae(prevs, prevs_hat, eps).mean()

+    return rae(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def rae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def rae(prevs_true, prevs_hat, eps=None):
     """Computes the absolute relative error between the two prevalence vectors.
      Relative absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -301,7 +741,7 @@
      where :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. `rae` is not defined in cases in which the true distribution
         contains zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the
@@ -310,12 +750,15 @@
     :return: relative absolute error
     """
     eps = __check_eps(eps)
-    prevs = smooth(prevs, eps)
+    prevs_true = smooth(prevs_true, eps)
     prevs_hat = smooth(prevs_hat, eps)
-    return (abs(prevs - prevs_hat) / prevs).mean(axis=-1)

+    return (abs(prevs_true - prevs_hat) / prevs_true).mean(axis=-1)

 
 
-
[docs]def nrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def nrae(prevs_true, prevs_hat, eps=None):
     """Computes the normalized absolute relative error between the two prevalence vectors.
      Relative absolute error between two prevalence vectors :math:`p` and :math:`\\hat{p}`
      is computed as
@@ -325,7 +768,7 @@
      and :math:`\\mathcal{Y}` are the classes of interest.
      The distributions are smoothed using the `eps` factor (see :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
     :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
     :param eps: smoothing factor. `nrae` is not defined in cases in which the true distribution
         contains zeros; `eps` is typically set to be :math:`\\frac{1}{2T}`, with :math:`T` the
@@ -334,18 +777,21 @@
     :return: normalized relative absolute error
     """
     eps = __check_eps(eps)
-    prevs = smooth(prevs, eps)
+    prevs_true = smooth(prevs_true, eps)
     prevs_hat = smooth(prevs_hat, eps)
-    min_p = prevs.min(axis=-1)
-    return (abs(prevs - prevs_hat) / prevs).sum(axis=-1)/(prevs.shape[-1]-1+(1-min_p)/min_p)

+    min_p = prevs_true.min(axis=-1)
+    return (abs(prevs_true - prevs_hat) / prevs_true).sum(axis=-1)/(prevs_true.shape[-1] - 1 + (1 - min_p) / min_p)

 
 
-
[docs]def mnrae(prevs, prevs_hat, eps=None):
+
+

+[docs]
+def mnrae(prevs_true, prevs_hat, eps=None):
     """Computes the mean normalized relative absolute error (see :meth:`quapy.error.nrae`) across
     the sample pairs. The distributions are smoothed using the `eps` factor (see
     :meth:`quapy.error.smooth`).
 
-    :param prevs: array-like of shape `(n_samples, n_classes,)` with the true
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true
         prevalence values
     :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
         prevalence values
@@ -355,10 +801,84 @@
         the environment variable `SAMPLE_SIZE` (which has thus to be set beforehand).
     :return: mean normalized relative absolute error
     """
-    return nrae(prevs, prevs_hat, eps).mean()

+    return nrae(prevs_true, prevs_hat, eps).mean()

 
 
-
[docs]def smooth(prevs, eps):
+
+

+[docs]
+def nmd(prevs_true, prevs_hat):
+    """
+    Computes the Normalized Match Distance; which is the Normalized Distance multiplied by the factor
+    `1/(n-1)` to guarantee the measure ranges between 0 (best prediction) and 1 (worst prediction).
+
+    :param prevs_true: array-like of shape `(n_classes,)` or `(n_instances, n_classes)`  with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` or `(n_instances, n_classes)` with the predicted prevalence values
+    :return: float in [0,1]
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    n = prevs_true.shape[-1]
+    return (1./(n-1))*np.mean(match_distance(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def bias_binary(prevs_true, prevs_hat):
+    """
+    Computes the (positive) bias in a binary problem. The bias is simply the difference between the
+    predicted positive value and the true positive value, so that a positive such value indicates the
+    prediction has positive bias (i.e., it tends to overestimate) the true value, and negative otherwise.
+    :math:`bias(p,\\hat{p})=\\hat{p}_1-p_1`,
+    :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_samples, n_classes,)` with the predicted
+        prevalence values
+    :return: binary bias
+    """
+    prevs_true = np.asarray(prevs_true)
+    prevs_hat = np.asarray(prevs_hat)
+    assert prevs_true.shape[-1] == 2 and prevs_true.shape[-1] == 2, f'bias_binary can only be applied to binary problems'
+    return prevs_hat[...,1]-prevs_true[...,1]

+
+
+
+

+[docs]
+def mean_bias_binary(prevs_true, prevs_hat):
+    """
+    Computes the mean of the (positive) bias in a binary problem.
+    :param prevs_true: array-like of shape `(n_classes,)` with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` with the predicted prevalence values
+    :return: mean binary bias
+    """
+    return np.mean(bias_binary(prevs_true, prevs_hat))

+
+
+
+

+[docs]
+def md(prevs_true, prevs_hat, ERROR_TOL=1E-3):
+    """
+    Computes the Match Distance, under the assumption that the cost in mistaking class i with class i+1 is 1 in
+    all cases.
+
+    :param prevs_true: array-like of shape `(n_classes,)` or `(n_instances, n_classes)`  with the true prevalence values
+    :param prevs_hat: array-like of shape `(n_classes,)` or `(n_instances, n_classes)` with the predicted prevalence values
+    :return: float
+    """
+    P = np.cumsum(prevs_true, axis=-1)
+    P_hat = np.cumsum(prevs_hat, axis=-1)
+    assert np.all(np.isclose(P_hat[..., -1], 1.0, rtol=ERROR_TOL)), \
+        'arg error in match_distance: the array does not represent a valid distribution'
+    distances = np.abs(P-P_hat)
+    return distances[..., :-1].sum(axis=-1)

+
+
+
+

+[docs]
+def smooth(prevs, eps):
     """ Smooths a prevalence distribution with :math:`\\epsilon` (`eps`) as:
     :math:`\\underline{p}(y)=\\frac{\\epsilon+p(y)}{\\epsilon|\\mathcal{Y}|+
     \\displaystyle\\sum_{y\\in \\mathcal{Y}}p(y)}`
@@ -367,11 +887,13 @@
     :param eps: smoothing factor
     :return: array-like of shape `(n_classes,)` with the smoothed distribution
     """
+    prevs = np.asarray(prevs)
     n_classes = prevs.shape[-1]
     return (prevs + eps) / (eps * n_classes + 1)

 
 
-def __check_eps(eps=None):
+
+def __check_eps(eps=None):
     if eps is None:
         sample_size = qp.environ['SAMPLE_SIZE']
         if sample_size is None:
@@ -381,8 +903,8 @@
 
 
 CLASSIFICATION_ERROR = {f1e, acce}
-QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld}
-QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld}
+QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld, msre, maitchisondist}
+QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld, sre, aitchisondist}
 QUANTIFICATION_ERROR_SMOOTH = {kld, nkld, rae, nrae, mkld, mnkld, mrae}
 CLASSIFICATION_ERROR_NAMES = {func.__name__ for func in CLASSIFICATION_ERROR}
 QUANTIFICATION_ERROR_NAMES = {func.__name__ for func in QUANTIFICATION_ERROR}
@@ -394,6 +916,9 @@
 f1_error = f1e
 acc_error = acce
 mean_absolute_error = mae
+squared_ratio_error = sre
+dist_aitchison = aitchisondist
+mean_dist_aitchison = maitchisondist
 absolute_error = ae
 mean_relative_absolute_error = mrae
 relative_absolute_error = rae
@@ -401,33 +926,79 @@
 normalized_relative_absolute_error = nrae
 mean_normalized_absolute_error = mnae
 mean_normalized_relative_absolute_error = mnrae
+normalized_match_distance = nmd
+match_distance = md
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - +
+
+ +
+ +
-
-
-
- -
- +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+
+ +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ +
+ + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/evaluation.html b/docs/build/html/_modules/quapy/evaluation.html index 56d34a5..4907b5b 100644 --- a/docs/build/html/_modules/quapy/evaluation.html +++ b/docs/build/html/_modules/quapy/evaluation.html @@ -1,23 +1,20 @@ + + - + - quapy.evaluation — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + quapy.evaluation — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation + + - - - - - - - - + + + + + @@ -43,7 +40,13 @@ @@ -71,16 +74,18 @@

Source code for quapy.evaluation

-from typing import Union, Callable, Iterable
-import numpy as np
-from tqdm import tqdm
-import quapy as qp
-from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol, IterateProtocol
-from quapy.method.base import BaseQuantifier
-import pandas as pd
+from typing import Union, Callable, Iterable
+import numpy as np
+from tqdm import tqdm
+import quapy as qp
+from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol, IterateProtocol
+from quapy.method.base import BaseQuantifier
+import pandas as pd
 
 
-
[docs]def prediction(
+

+[docs]
+def prediction(
         model: BaseQuantifier,
         protocol: AbstractProtocol,
         aggr_speedup: Union[str, bool] = 'auto',
@@ -118,7 +123,7 @@
         # checks whether the prediction can be made more efficiently; this check consists in verifying if the model is
         # of type aggregative, if the protocol is based on LabelledCollection, and if the total number of documents to
         # classify using the protocol would exceed the number of test documents in the original collection
-        from quapy.method.aggregative import AggregativeQuantifier
+        from quapy.method.aggregative import AggregativeQuantifier
         if isinstance(model, AggregativeQuantifier) and isinstance(protocol, OnLabelledCollectionProtocol):
             if aggr_speedup == 'force':
                 apply_optimization = True
@@ -136,10 +141,11 @@
         protocol_with_predictions = protocol.on_preclassified_instances(pre_classified)
         return __prediction_helper(model.aggregate, protocol_with_predictions, verbose)
     else:
-        return __prediction_helper(model.quantify, protocol, verbose)

+        return __prediction_helper(model.predict, protocol, verbose)

 
 
-def __prediction_helper(quantification_fn, protocol: AbstractProtocol, verbose=False):
+
+def __prediction_helper(quantification_fn, protocol: AbstractProtocol, verbose=False):
     true_prevs, estim_prevs = [], []
     for sample_instances, sample_prev in tqdm(protocol(), total=protocol.total(), desc='predicting') if verbose else protocol():
         estim_prevs.append(quantification_fn(sample_instances))
@@ -151,7 +157,9 @@
     return true_prevs, estim_prevs
 
 
-
[docs]def evaluation_report(model: BaseQuantifier,
+

+[docs]
+def evaluation_report(model: BaseQuantifier,
                       protocol: AbstractProtocol,
                       error_metrics: Iterable[Union[str,Callable]] = 'mae',
                       aggr_speedup: Union[str, bool] = 'auto',
@@ -182,7 +190,8 @@
     return _prevalence_report(true_prevs, estim_prevs, error_metrics)

 
 
-def _prevalence_report(true_prevs, estim_prevs, error_metrics: Iterable[Union[str, Callable]] = 'mae'):
+
+def _prevalence_report(true_prevs, estim_prevs, error_metrics: Iterable[Union[str, Callable]] = 'mae'):
 
     if isinstance(error_metrics, str):
         error_metrics = [error_metrics]
@@ -203,7 +212,9 @@
     return df
 
 
-
[docs]def evaluate(
+

+[docs]
+def evaluate(
         model: BaseQuantifier,
         protocol: AbstractProtocol,
         error_metric: Union[str, Callable],
@@ -235,7 +246,10 @@
     return error_metric(true_prevs, estim_prevs)

 
 
-
[docs]def evaluate_on_samples(
+
+

+[docs]
+def evaluate_on_samples(
         model: BaseQuantifier,
         samples: Iterable[qp.data.LabelledCollection],
         error_metric: Union[str, Callable],
@@ -259,6 +273,7 @@
 
 
 
+
diff --git a/docs/build/html/_modules/quapy/functional.html b/docs/build/html/_modules/quapy/functional.html index 1b02248..ce5d72e 100644 --- a/docs/build/html/_modules/quapy/functional.html +++ b/docs/build/html/_modules/quapy/functional.html @@ -1,125 +1,463 @@ + - - - - - quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.functional — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.functional

-import itertools
-from collections import defaultdict
-from typing import Union, Callable
+import warnings
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from functools import lru_cache
+from typing import Literal, Union, Callable
+from numpy.typing import ArrayLike
 
-import scipy
-import numpy as np
+import scipy
+import numpy as np
+
+import quapy as qp
 
 
-
[docs]def prevalence_linspace(n_prevalences=21, repeats=1, smooth_limits_epsilon=0.01):
+# ------------------------------------------------------------------------------------------
+# General utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def classes_from_labels(labels):
     """
-    Produces an array of uniformly separated values of prevalence.
-    By default, produces an array of 21 prevalence values, with
-    step 0.05 and with the limits smoothed, i.e.:
-    [0.01, 0.05, 0.10, 0.15, ..., 0.90, 0.95, 0.99]
-
-    :param n_prevalences: the number of prevalence values to sample from the [0,1] interval (default 21)
-    :param repeats: number of times each prevalence is to be repeated (defaults to 1)
-    :param smooth_limits_epsilon: the quantity to add and subtract to the limits 0 and 1
-    :return: an array of uniformly separated prevalence values
+    Obtains a np.ndarray with the (sorted) classes
+    :param labels: array-like with the instances' labels
+    :return: a sorted np.ndarray with the class labels
     """
-    p = np.linspace(0., 1., num=n_prevalences, endpoint=True)
-    p[0] += smooth_limits_epsilon
-    p[-1] -= smooth_limits_epsilon
-    if p[0] > p[1]:
-        raise ValueError(f'the smoothing in the limits is greater than the prevalence step')
-    if repeats > 1:
-        p = np.repeat(p, repeats)
-    return p

+    classes = np.unique(labels)
+    classes.sort()
+    return classes

 
 
-
[docs]def prevalence_from_labels(labels, classes):
+
+

+[docs]
+def num_classes_from_labels(labels):
     """
-    Computed the prevalence values from a vector of labels.
+    Obtains the number of classes from an array-like of instance's labels
+    :param labels: array-like with the instances' labels
+    :return: int, the number of classes
+    """
+    return len(classes_from_labels(labels))

 
-    :param labels: array-like of shape `(n_instances)` with the label for each instance
+
+# ------------------------------------------------------------------------------------------
+# Counter utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def counts_from_labels(labels: ArrayLike, classes: ArrayLike) -> np.ndarray:
+    """
+    Computes the raw count values from a vector of labels.
+
+    :param labels: array-like of shape `(n_instances,)` with the label for each instance
     :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
         some classes have no examples.
-    :return: an ndarray of shape `(len(classes))` with the class prevalence values
+    :return: ndarray of shape `(len(classes),)` with the raw counts for each class, in the same order
+        as they appear in `classes`
     """
-    if labels.ndim != 1:
+    if np.asarray(labels).ndim != 1:
         raise ValueError(f'param labels does not seem to be a ndarray of label predictions')
     unique, counts = np.unique(labels, return_counts=True)
     by_class = defaultdict(lambda:0, dict(zip(unique, counts)))
-    prevalences = np.asarray([by_class[class_] for class_ in classes], dtype=float)
-    prevalences /= prevalences.sum()
+    counts = np.asarray([by_class[class_] for class_ in classes], dtype=int)
+    return counts

+
+
+
+

+[docs]
+def prevalence_from_labels(labels: ArrayLike, classes: ArrayLike):
+    """
+    Computes the prevalence values from a vector of labels.
+
+    :param labels: array-like of shape `(n_instances,)` with the label for each instance
+    :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
+        some classes have no examples.
+    :return: ndarray of shape `(len(classes),)` with the class proportions for each class, in the same order
+        as they appear in `classes`
+    """
+    counts = counts_from_labels(labels, classes)
+    prevalences = counts.astype(float) / np.sum(counts)
     return prevalences

 
 
-
[docs]def prevalence_from_probabilities(posteriors, binarize: bool = False):
+
+

+[docs]
+def prevalence_from_probabilities(posteriors: ArrayLike, binarize: bool = False):
     """
     Returns a vector of prevalence values from a matrix of posterior probabilities.
 
@@ -128,8 +466,9 @@
         converting the vectors of posterior probabilities into class indices, by taking the argmax).
     :return: array of shape `(n_classes,)` containing the prevalence values
     """
+    posteriors = np.asarray(posteriors)
     if posteriors.ndim != 2:
-        raise ValueError(f'param posteriors does not seem to be a ndarray of posteior probabilities')
+        raise ValueError(f'param posteriors does not seem to be a ndarray of posterior probabilities')
     if binarize:
         predictions = np.argmax(posteriors, axis=-1)
         return prevalence_from_labels(predictions, np.arange(posteriors.shape[1]))
@@ -139,25 +478,316 @@
         return prevalences

 
 
-
[docs]def as_binary_prevalence(positive_prevalence: Union[float, np.ndarray], clip_if_necessary=False):
+
+

+[docs]
+def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1) -> int:
+    """
+    Computes the number of valid prevalence combinations in the n_classes-dimensional simplex if `n_prevpoints` equally
+    distant prevalence values are generated and `n_repeats` repetitions are requested.
+    The computation comes down to calculating:
+
+    .. math::
+        \\binom{N+C-1}{C-1} \\times r
+
+    where `N` is `n_prevpoints-1`, i.e., the number of probability mass blocks to allocate, `C` is the number of
+    classes, and `r` is `n_repeats`. This solution comes from the
+    `Stars and Bars <https://brilliant.org/wiki/integer-equations-star-and-bars/>`_ problem.
+
+    :param int n_classes: number of classes
+    :param int n_prevpoints: number of prevalence points.
+    :param int n_repeats: number of repetitions for each prevalence combination
+    :return: The number of possible combinations. For example, if `n_classes`=2, `n_prevpoints`=5, `n_repeats`=1,
+        then the number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25],
+        and [1.0,0.0]
+    """
+    N = n_prevpoints-1
+    C = n_classes
+    r = n_repeats
+    return int(scipy.special.binom(N + C - 1, C - 1) * r)

+
+
+
+

+[docs]
+def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1) -> int:
+    """
+    Searches for the largest number of (equidistant) prevalence points to define for each of the `n_classes` classes so
+    that the number of valid prevalence values generated as combinations of prevalence points (points in a
+    `n_classes`-dimensional simplex) do not exceed combinations_budget.
+
+    :param int combinations_budget: maximum number of combinations allowed
+    :param int n_classes: number of classes
+    :param int n_repeats: number of repetitions for each prevalence combination
+    :return: the largest number of prevalence points that generate less than combinations_budget valid prevalences
+    """
+    assert n_classes > 0 and n_repeats > 0 and combinations_budget > 0, 'parameters must be positive integers'
+    n_prevpoints = 1
+    while True:
+        combinations = num_prevalence_combinations(n_prevpoints, n_classes, n_repeats)
+        if combinations > combinations_budget:
+            return n_prevpoints-1
+        else:
+            n_prevpoints += 1

+
+
+
+# ------------------------------------------------------------------------------------------
+# Prevalence vectors
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def as_binary_prevalence(positive_prevalence: Union[float, ArrayLike], clip_if_necessary: bool=False) -> np.ndarray:
     """
     Helper that, given a float representing the prevalence for the positive class, returns a np.ndarray of two
     values representing a binary distribution.
 
-    :param positive_prevalence: prevalence for the positive class
-    :param clip_if_necessary: if True, clips the value in [0,1] in order to guarantee the resulting distribution
+    :param positive_prevalence: float or array-like of floats with the prevalence for the positive class
+    :param bool clip_if_necessary: if True, clips the value in [0,1] in order to guarantee the resulting distribution
         is valid. If False, it then checks that the value is in the valid range, and raises an error if not.
     :return: np.ndarray of shape `(2,)`
     """
+    positive_prevalence = np.asarray(positive_prevalence, float)
     if clip_if_necessary:
         positive_prevalence = np.clip(positive_prevalence, 0, 1)
     else:
-        assert 0 <= positive_prevalence <= 1, 'the value provided is not a valid prevalence for the positive class'
+        assert np.logical_and(0 <= positive_prevalence, positive_prevalence  <= 1).all(), \
+            'the value provided is not a valid prevalence for the positive class'
     return np.asarray([1-positive_prevalence, positive_prevalence]).T

 
 
 
-
[docs]def HellingerDistance(P, Q) -> float:
+

+[docs]
+def strprev(prevalences: ArrayLike, prec: int=3) -> str:
+    """
+    Returns a string representation for a prevalence vector. E.g.,
+
+    >>> strprev([1/3, 2/3], prec=2)
+    >>> '[0.33, 0.67]'
+
+    :param prevalences: array-like of prevalence values
+    :param prec: int, indicates the float precision (number of decimal values to print)
+    :return: string
+    """
+    return '['+ ', '.join([f'{p:.{prec}f}' for p in prevalences]) + ']'

+
+
+
+

+[docs]
+def check_prevalence_vector(prevalences: ArrayLike, raise_exception: bool=False, tolerance: float=1e-08, aggr=True):
+    """
+    Checks that `prevalences` is a valid prevalence vector, i.e., it contains values in [0,1] and
+    the values sum up to 1. In other words, verifies that the `prevalences` vectors lies in the
+    probability simplex.
+
+    :param ArrayLike prevalences: the prevalence vector, or vectors, to check
+    :param bool raise_exception: whether to raise an exception if the vector (or any of the vectors) does
+        not lie in the simplex (default False)
+    :param float tolerance: error tolerance for the check `sum(prevalences) - 1 = 0`
+    :param bool aggr: if True (default) returns one single bool (True if all prevalence vectors are valid,
+        False otherwise), if False returns an array of bool, one for each prevalence vector
+    :return: a single bool True if `prevalences` is a vector of prevalence values that lies on the simplex,
+        or False otherwise; alternatively, if `prevalences` is a matrix of shape `(num_vectors, n_classes,)`
+        then it returns one such bool for each prevalence vector
+    """
+    prevalences = np.asarray(prevalences)
+
+    all_positive = prevalences>=0
+    if not all_positive.all():
+        if raise_exception:
+            raise ValueError('some prevalence vectors contain negative numbers; '
+                             'consider using the qp.functional.normalize_prevalence with '
+                             'any method from ["clip", "mapsimplex", "softmax"]')
+
+    all_close_1 = np.isclose(prevalences.sum(axis=-1), 1, atol=tolerance)
+    if not all_close_1.all():
+        if raise_exception:
+            raise ValueError('some prevalence vectors do not sum up to 1; '
+                             'consider using the qp.functional.normalize_prevalence with '
+                             'any method from ["l1", "clip", "mapsimplex", "softmax"]')
+
+    valid = np.logical_and(all_positive.all(axis=-1), all_close_1)
+    if aggr:
+        return valid.all()
+    else:
+        return valid

+
+
+
+

+[docs]
+def uniform_prevalence(n_classes):
+    """
+    Returns a vector representing the uniform distribution for `n_classes`
+
+    :param n_classes: number of classes
+    :return: np.ndarray with all values 1/n_classes
+    """
+    assert isinstance(n_classes, int) and n_classes>0, \
+        (f'param {n_classes} not understood; must be a positive integer representing the '
+         f'number of classes ')
+    return np.full(shape=n_classes, fill_value=1./n_classes)

+
+
+
+

+[docs]
+def normalize_prevalence(prevalences: ArrayLike, method='l1'):
+    """
+    Normalizes a vector or matrix of prevalence values. The normalization consists of applying a L1 normalization in
+    cases in which the prevalence values are not all-zeros, and to convert the prevalence values into `1/n_classes` in
+    cases in which all values are zero.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :param str method: indicates the normalization method to employ, options are:
+
+        * `l1`: applies L1 normalization (default); a 0 vector is mapped onto the uniform prevalence
+        * `clip`: clip values in [0,1] and then rescales so that the L1 norm is 1
+        * `mapsimplex`: projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `softmax`: applies softmax to all vectors
+        * `condsoftmax`: applies softmax only to invalid prevalence vectors
+
+    :return: a normalized vector or matrix of prevalence values
+    """
+    if method in ['none', None]:
+        return prevalences
+
+    prevalences = np.asarray(prevalences, dtype=float)
+
+    if method=='l1':
+        normalized = l1_norm(prevalences)
+        check_prevalence_vector(normalized, raise_exception=True)
+    elif method=='clip':
+        normalized = clip(prevalences)  # no need to check afterwards
+    elif method=='mapsimplex':
+        normalized = projection_simplex_sort(prevalences)
+    elif method=='softmax':
+        normalized = softmax(prevalences)
+    elif method=='condsoftmax':
+        normalized = condsoftmax(prevalences)
+    else:
+        raise ValueError(f'unknown {method=}, valid ones are ["l1", "clip", "mapsimplex", "softmax", "condsoftmax"]')
+
+    return normalized

+
+
+
+

+[docs]
+def l1_norm(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies L1 normalization to the `unnormalized_arr` so that it becomes a valid prevalence
+    vector. Zero vectors are mapped onto the uniform distribution. Raises an exception if
+    the resulting vectors are not valid distributions. This may happen when the original
+    prevalence vectors contain negative values. Use the `clip` normalization function
+    instead to avoid this possibility.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    n_classes = prevalences.shape[-1]
+    accum = prevalences.sum(axis=-1, keepdims=True)
+    prevalences = np.true_divide(prevalences, accum, where=accum > 0, out=None)
+    allzeros = accum.flatten() == 0
+    if any(allzeros):
+        if prevalences.ndim == 1:
+            prevalences = np.full(shape=n_classes, fill_value=1. / n_classes)
+        else:
+            prevalences[allzeros] = np.full(shape=n_classes, fill_value=1. / n_classes)
+    return prevalences

+
+
+
+

+[docs]
+def clip(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Clips the values in [0,1] and then applies the L1 normalization.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    clipped = np.clip(prevalences, 0, 1)
+    normalized = l1_norm(clipped)
+    return normalized

+
+
+
+

+[docs]
+def projection_simplex_sort(unnormalized_arr: ArrayLike) -> np.ndarray:
+    """Projects a point onto the probability simplex.
+
+    The code is adapted from Mathieu Blondel's BSD-licensed
+    `implementation <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+    (see function `projection_simplex_sort` in their repo) which is accompanying the paper
+
+    Mathieu Blondel, Akinori Fujino, and Naonori Ueda.
+    Large-scale Multiclass Support Vector Machine Training via Euclidean Projection onto the Simplex,
+    ICPR 2014, `URL <http://www.mblondel.org/publications/mblondel-icpr2014.pdf>`_
+
+    :param `unnormalized_arr`: point in n-dimensional space, shape `(n,)`
+    :return: projection of `unnormalized_arr` onto the (n-1)-dimensional probability simplex, shape `(n,)`
+    """
+    unnormalized_arr = np.asarray(unnormalized_arr)
+    n = len(unnormalized_arr)
+    u = np.sort(unnormalized_arr)[::-1]
+    cssv = np.cumsum(u) - 1.0
+    ind = np.arange(1, n + 1)
+    cond = u - cssv / ind > 0
+    rho = ind[cond][-1]
+    theta = cssv[cond][-1] / float(rho)
+    return np.maximum(unnormalized_arr - theta, 0)

+
+
+
+

+[docs]
+def softmax(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies the softmax function to all vectors even if the original vectors were valid distributions.
+    If you want to leave valid vectors untouched, use condsoftmax instead.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    normalized = scipy.special.softmax(prevalences, axis=-1)
+    return normalized

+
+
+
+

+[docs]
+def condsoftmax(prevalences: ArrayLike) -> np.ndarray:
+    """
+    Applies the softmax function only to vectors that do not represent valid distributions.
+
+    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
+    :return: np.ndarray representing a valid distribution
+    """
+    invalid_idx = ~ check_prevalence_vector(prevalences, aggr=False, raise_exception=False)
+    if isinstance(invalid_idx, np.bool_) and invalid_idx:
+        # only one vector
+        normalized = scipy.special.softmax(prevalences)
+    else:
+        prevalences = np.copy(prevalences)
+        prevalences[invalid_idx] = scipy.special.softmax(prevalences[invalid_idx], axis=-1)
+        normalized = prevalences
+    return normalized

+
+
+
+# ------------------------------------------------------------------------------------------
+# Divergences
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def HellingerDistance(P: np.ndarray, Q: np.ndarray) -> float:
     """
     Computes the Hellingher Distance (HD) between (discretized) distributions `P` and `Q`.
     The HD for two discrete distributions of `k` bins is defined as:
@@ -172,7 +802,10 @@
     return np.sqrt(np.sum((np.sqrt(P) - np.sqrt(Q))**2))

 
 
-
[docs]def TopsoeDistance(P, Q, epsilon=1e-20):
+
+

+[docs]
+def TopsoeDistance(P: np.ndarray, Q: np.ndarray, epsilon: float=1e-20):
     """
     Topsoe distance between two (discretized) distributions `P` and `Q`.
     The Topsoe distance for two discrete distributions of `k` bins is defined as:
@@ -186,9 +819,184 @@
     :return: float
     """
     return np.sum(P*np.log((2*P+epsilon)/(P+Q+epsilon)) + Q*np.log((2*Q+epsilon)/(P+Q+epsilon)))

-                  
 
-
[docs]def uniform_prevalence_sampling(n_classes, size=1):
+
+
+

+[docs]
+def get_divergence(divergence: Union[str, Callable]):
+    """
+    Guarantees that the divergence received as argument is a function. That is, if this argument is already
+    a callable, then it is returned, if it is instead a string, then tries to instantiate the corresponding
+    divergence from the string name.
+
+    :param divergence: callable or string indicating the name of the divergence function
+    :return: callable
+    """
+    if isinstance(divergence, str):
+        if divergence=='HD':
+            return HellingerDistance
+        elif divergence=='topsoe':
+            return TopsoeDistance
+        else:
+            raise ValueError(f'unknown divergence {divergence}')
+    elif callable(divergence):
+        return divergence
+    else:
+        raise ValueError(f'argument "divergence" not understood; use a str or a callable function')

+
+
+
+# ------------------------------------------------------------------------------------------
+# Solvers
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def argmin_prevalence(loss: Callable,
+                      n_classes: int,
+                      method: Literal["optim_minimize", "linear_search", "ternary_search"]='optim_minimize'):
+    """
+    Searches for the prevalence vector that minimizes a loss function.
+
+    :param loss: callable, the function to minimize
+    :param n_classes: int, number of classes
+    :param method: string indicating the search strategy. Possible values are::
+        'optim_minimize': uses scipy.optim
+        'linear_search': carries out a linear search for binary problems in the space [0, 0.01, 0.02, ..., 1]
+        'ternary_search': carries out a ternary search for binary problems in the interval [0,1]
+    :return: np.ndarray, a prevalence vector
+    """
+    if method == 'optim_minimize':
+        return optim_minimize(loss, n_classes)
+    elif method == 'linear_search':
+        return linear_search(loss, n_classes)
+    elif method == 'ternary_search':
+        return ternary_search(loss, n_classes)
+    else:
+        raise NotImplementedError()

+
+
+
+

+[docs]
+def optim_minimize(loss: Callable, n_classes: int, return_loss=False):
+    """
+    Searches for the optimal prevalence values, i.e., an `n_classes`-dimensional vector of the (`n_classes`-1)-simplex
+    that yields the smallest lost. This optimization is carried out by means of a constrained search using scipy's
+    SLSQP routine.
+
+    :param loss: (callable) the function to minimize
+    :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector
+    :param return_loss: bool, if True, returns also the value of the loss (default is False).
+    :return: (ndarray) the best prevalence vector found or a tuple which also contains the value of the loss
+        if return_loss=True
+    """
+    from scipy import optimize
+
+    # the initial point is set as the uniform distribution
+    uniform_distribution = uniform_prevalence(n_classes=n_classes)
+
+    # solutions are bounded to those contained in the unit-simplex
+    bounds = tuple((0, 1) for _ in range(n_classes))  # values in [0,1]
+    constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)})  # values summing up to 1
+    r = optimize.minimize(loss, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints)
+    
+    if return_loss:
+        return r.x, r.fun
+    else:
+        return r.x

+
+
+
+
+
+
+
+
+
+
+
+# ------------------------------------------------------------------------------------------
+# Sampling utils
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+def prevalence_linspace(grid_points:int=21, repeats:int=1, smooth_limits_epsilon:float=0.01) -> np.ndarray:
+    """
+    Produces an array of uniformly separated values of prevalence.
+    By default, produces an array of 21 prevalence values, with
+    step 0.05 and with the limits smoothed, i.e.:
+    [0.01, 0.05, 0.10, 0.15, ..., 0.90, 0.95, 0.99]
+
+    :param grid_points: the number of prevalence values to sample from the [0,1] interval (default 21)
+    :param repeats: number of times each prevalence is to be repeated (defaults to 1)
+    :param smooth_limits_epsilon: the quantity to add and subtract to the limits 0 and 1
+    :return: an array of uniformly separated prevalence values
+    """
+    p = np.linspace(0., 1., num=grid_points, endpoint=True)
+    p[0] += smooth_limits_epsilon
+    p[-1] -= smooth_limits_epsilon
+    if p[0] > p[1]:
+        raise ValueError(f'the smoothing in the limits is greater than the prevalence step')
+    if repeats > 1:
+        p = np.repeat(p, repeats)
+    return p

+
+
+
+

+[docs]
+def uniform_prevalence_sampling(n_classes: int, size: int=1) -> np.ndarray:
     """
     Implements the `Kraemer algorithm <http://www.cs.cmu.edu/~nasmith/papers/smith+tromble.tr04.pdf>`_
     for sampling uniformly at random from the unit simplex. This implementation is adapted from this
@@ -214,24 +1022,17 @@
     return u

 
 
+
 uniform_simplex_sampling = uniform_prevalence_sampling
 
 
-
[docs]def strprev(prevalences, prec=3):
-    """
-    Returns a string representation for a prevalence vector. E.g.,
+# ------------------------------------------------------------------------------------------
+# Adjustment
+# ------------------------------------------------------------------------------------------
 
-    >>> strprev([1/3, 2/3], prec=2)
-    >>> '[0.33, 0.67]'
-
-    :param prevalences: a vector of prevalence values
-    :param prec: float precision
-    :return: string
-    """
-    return '['+ ', '.join([f'{p:.{prec}f}' for p in prevalences]) + ']'

-
-
-
[docs]def adjusted_quantification(prevalence_estim, tpr, fpr, clip=True):
+

+[docs]
+def solve_adjustment_binary(prevalence_estim: ArrayLike, tpr: float, fpr: float, clip: bool=True):
     """
     Implements the adjustment of ACC and PACC for the binary case. The adjustment for a prevalence estimate of the
     positive class `p` comes down to computing:
@@ -239,10 +1040,10 @@
     .. math::
         ACC(p) = \\frac{ p - fpr }{ tpr - fpr }
 
-    :param prevalence_estim: float, the estimated value for the positive class
-    :param tpr: float, the true positive rate of the classifier
-    :param fpr: float, the false positive rate of the classifier
-    :param clip: set to True (default) to clip values that might exceed the range [0,1]
+    :param float prevalence_estim: the estimated value for the positive class (`p` in the formula)
+    :param float tpr: the true positive rate of the classifier
+    :param float fpr: the false positive rate of the classifier
+    :param bool clip: set to True (default) to clip values that might exceed the range [0,1]
     :return: float, the adjusted count
     """
 
@@ -255,214 +1056,291 @@
     return adjusted

 
 
-
[docs]def normalize_prevalence(prevalences):
-    """
-    Normalize a vector or matrix of prevalence values. The normalization consists of applying a L1 normalization in
-    cases in which the prevalence values are not all-zeros, and to convert the prevalence values into `1/n_classes` in
-    cases in which all values are zero.
 
-    :param prevalences: array-like of shape `(n_classes,)` or of shape `(n_samples, n_classes,)` with prevalence values
-    :return: a normalized vector or matrix of prevalence values
+

+[docs]
+def solve_adjustment(
+    class_conditional_rates: np.ndarray,
+    unadjusted_counts: np.ndarray,
+    method: Literal["inversion", "invariant-ratio"],
+    solver: Literal["exact", "minimize", "exact-raise", "exact-cc"]) -> np.ndarray:
+    """
+    Function that tries to solve for :math:`p` the equation :math:`q = M p`, where :math:`q` is the vector of
+    `unadjusted counts` (as estimated, e.g., via classify and count) with :math:`q_i` an estimate of
+    :math:`P(\hat{Y}=y_i)`, and where :math:`M` is the matrix of `class-conditional rates` with :math:`M_{ij}` an
+    estimate of :math:`P(\hat{Y}=y_i|Y=y_j)`.
+
+    :param class_conditional_rates: array of shape `(n_classes, n_classes,)` with entry `(i,j)` being the estimate
+        of :math:`P(\hat{Y}=y_i|Y=y_j)`, that is, the probability that an instance that belongs to class :math:`y_j`
+        ends up being classified as belonging to class :math:`y_i`
+
+    :param unadjusted_counts: array of shape `(n_classes,)` containing the unadjusted prevalence values (e.g., as
+        estimated by CC or PCC)
+
+    :param str method: indicates the adjustment method to be used. Valid options are:
+
+        * `inversion`: tries to solve the equation :math:`q = M p` as :math:`p = M^{-1} q` where
+          :math:`M^{-1}` is the matrix inversion of :math:`M`. This inversion may not exist in
+          degenerated cases.
+        * `invariant-ratio`: invariant ratio estimator of `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation in :math:`M` with the normalization condition (i.e., that the sum of
+          all prevalence values must equal 1).
+
+    :param str solver: the method to use for solving the system of linear equations. Valid options are:
+
+        * `exact-raise`: tries to solve the system using matrix inversion. Raises an error if the matrix has rank
+          strictly lower than `n_classes`.
+        * `exact-cc`: if the matrix is not full rank, returns :math:`q` (i.e., the unadjusted counts) as the estimates
+        * `exact`: deprecated, defaults to 'exact-cc' (will be removed in future versions)
+        * `minimize`: minimizes a loss, so the solution always exists
     """
-    prevalences = np.asarray(prevalences)
-    n_classes = prevalences.shape[-1]
-    accum = prevalences.sum(axis=-1, keepdims=True)
-    prevalences = np.true_divide(prevalences, accum, where=accum>0)
-    allzeros = accum.flatten()==0
-    if any(allzeros):
-        if prevalences.ndim == 1:
-            prevalences = np.full(shape=n_classes, fill_value=1./n_classes)
-        else:
-            prevalences[accum.flatten()==0] = np.full(shape=n_classes, fill_value=1./n_classes)
-    return prevalences

+    if solver == "exact":
+        warnings.warn(
+            "The 'exact' solver is deprecated. Use 'exact-raise' or 'exact-cc'", DeprecationWarning, stacklevel=2)
+        solver = "exact-cc"
+
+    A = np.asarray(class_conditional_rates, dtype=float)
+    B = np.asarray(unadjusted_counts, dtype=float)
+
+    if method == "inversion":
+        pass  # We leave A and B unchanged
+    elif method == "invariant-ratio":
+        # Change the last equation to replace it with the normalization condition;
+        # copy first so this does not mutate the caller's arrays (np.asarray above
+        # returns the same object, not a copy, when the input is already float64)
+        A = A.copy()
+        B = B.copy()
+        A[-1, :] = 1.0
+        B[-1] = 1.0
+    else:
+        raise ValueError(f"unknown {method=}")
+
+    if solver == "minimize":
+        def loss(prev):
+            return np.linalg.norm(A @ prev - B)
+        return optim_minimize(loss, n_classes=A.shape[0])
+    elif solver in ["exact-raise", "exact-cc"]:
+        # Solvers based on matrix inversion, so we use try/except block
+        try:
+            return np.linalg.solve(A, B)
+        except np.linalg.LinAlgError:
+            # The matrix is not invertible.
+            # Depending on the solver, we either raise an error
+            # or return the classifier predictions without adjustment
+            if solver == "exact-raise":
+                raise
+            elif solver == "exact-cc":
+                return unadjusted_counts
+            else:
+                raise ValueError(f"Solver {solver} not known.")
+    else:
+        raise ValueError(f'unknown {solver=}')

 
 
-def __num_prevalence_combinations_depr(n_prevpoints:int, n_classes:int, n_repeats:int=1):
+
+# ------------------------------------------------------------------------------------------
+# Transformations from Compositional analysis
+# ------------------------------------------------------------------------------------------
+
+

+[docs]
+class CompositionalTransformation(ABC):
     """
-    Computes the number of prevalence combinations in the n_classes-dimensional simplex if `nprevpoints` equally distant
-    prevalence values are generated and `n_repeats` repetitions are requested.
-
-    :param n_classes: integer, number of classes
-    :param n_prevpoints: integer, number of prevalence points.
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: The number of possible combinations. For example, if n_classes=2, n_prevpoints=5, n_repeats=1, then the
-        number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25], and [1.0,0.0]
+    Abstract class of transformations for compositional data.
     """
-    __cache={}
-    def __f(nc,np):
-        if (nc,np) in __cache:  # cached result
-            return __cache[(nc,np)]
-        if nc==1:  # stop condition
-            return 1
-        else:  # recursive call
-            x = sum([__f(nc-1, np-i) for i in range(np)])
-            __cache[(nc,np)] = x
-            return x
-    return __f(n_classes, n_prevpoints) * n_repeats
+
+    EPSILON = 1e-12
+
+    @abstractmethod
+    def __call__(self, X):
+        ...
+
+

+[docs]
+    @abstractmethod
+    def inverse(self, Z):
+        ...

+

 
 
-
[docs]def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1):
+
+

+[docs]
+class CLRtransformation(CompositionalTransformation):
     """
-    Computes the number of valid prevalence combinations in the n_classes-dimensional simplex if `n_prevpoints` equally
-    distant prevalence values are generated and `n_repeats` repetitions are requested.
-    The computation comes down to calculating:
-
-    .. math::
-        \\binom{N+C-1}{C-1} \\times r
-
-    where `N` is `n_prevpoints-1`, i.e., the number of probability mass blocks to allocate, `C` is the number of
-    classes, and `r` is `n_repeats`. This solution comes from the
-    `Stars and Bars <https://brilliant.org/wiki/integer-equations-star-and-bars/>`_ problem.
-
-    :param n_classes: integer, number of classes
-    :param n_prevpoints: integer, number of prevalence points.
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: The number of possible combinations. For example, if n_classes=2, n_prevpoints=5, n_repeats=1, then the
-        number of possible combinations are 5, i.e.: [0,1], [0.25,0.75], [0.50,0.50], [0.75,0.25], and [1.0,0.0]
+    Centered log-ratio (CLR) transformation.
     """
-    N = n_prevpoints-1
-    C = n_classes
-    r = n_repeats
-    return int(scipy.special.binom(N + C - 1, C - 1) * r)

+
+    def __call__(self, X):
+        X = np.asarray(X)
+        X = qp.error.smooth(X, self.EPSILON)
+        geometric_mean = np.exp(np.mean(np.log(X), axis=-1, keepdims=True))
+        return np.log(X / geometric_mean)
+
+

+[docs]
+    def inverse(self, Z):
+        return scipy.special.softmax(Z, axis=-1)

+

 
 
-
[docs]def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1):
+
+

+[docs]
+class ILRtransformation(CompositionalTransformation):
     """
-    Searches for the largest number of (equidistant) prevalence points to define for each of the `n_classes` classes so
-    that the number of valid prevalence values generated as combinations of prevalence points (points in a
-    `n_classes`-dimensional simplex) do not exceed combinations_budget.
-
-    :param combinations_budget: integer, maximum number of combinations allowed
-    :param n_classes: integer, number of classes
-    :param n_repeats: integer, number of repetitions for each prevalence combination
-    :return: the largest number of prevalence points that generate less than combinations_budget valid prevalences
+    Isometric log-ratio (ILR) transformation.
     """
-    assert n_classes > 0 and n_repeats > 0 and combinations_budget > 0, 'parameters must be positive integers'
-    n_prevpoints = 1
-    while True:
-        combinations = num_prevalence_combinations(n_prevpoints, n_classes, n_repeats)
-        if combinations > combinations_budget:
-            return n_prevpoints-1
-        else:
-            n_prevpoints += 1

+
+    def __call__(self, X):
+        X = np.asarray(X)
+        X = qp.error.smooth(X, self.EPSILON)
+        basis = self.get_V(X.shape[-1])
+        return np.log(X) @ basis.T
+
+

+[docs]
+    def inverse(self, Z):
+        Z = np.asarray(Z)
+        basis = self.get_V(Z.shape[-1] + 1)
+        logp = Z @ basis
+        p = np.exp(logp)
+        return p / np.sum(p, axis=-1, keepdims=True)

 
 
-
[docs]def check_prevalence_vector(p, raise_exception=False, toleranze=1e-08):
+

+[docs]
+    @lru_cache(maxsize=None)
+    def get_V(self, k):
+        helmert = np.zeros((k, k))
+        for i in range(1, k):
+            helmert[i, :i] = 1
+            helmert[i, i] = -i
+            helmert[i] = helmert[i] / np.sqrt(i * (i + 1))
+        return helmert[1:, :]

+

+
+
+
+

+[docs]
+def normalized_entropy(p):
     """
-    Checks that p is a valid prevalence vector, i.e., that it contains values in [0,1] and that the values sum up to 1.
+    Computes the normalized Shannon entropy of a prevalence vector.
 
-    :param p: the prevalence vector to check
-    :return: True if `p` is valid, False otherwise
+    :param p: array-like prevalence vector summing to 1
+    :return: float in [0,1]
     """
     p = np.asarray(p)
-    if not all(p>=0):
-        if raise_exception:
-            raise ValueError('the prevalence vector contains negative numbers')
-        return False
-    if not all(p<=1):
-        if raise_exception:
-            raise ValueError('the prevalence vector contains values >1')
-        return False
-    if not np.isclose(p.sum(), 1, atol=toleranze):
-        if raise_exception:
-            raise ValueError('the prevalence vector does not sum up to 1')
-        return False
-    return True

+    entropy = scipy.stats.entropy(p)
+    max_entropy = np.log(len(p))
+    return np.clip(entropy / max_entropy, 0, 1)

 
 
-
[docs]def get_divergence(divergence: Union[str, Callable]):
-    if isinstance(divergence, str):
-        if divergence=='HD':
-            return HellingerDistance
-        elif divergence=='topsoe':
-            return TopsoeDistance
-        else:
-            raise ValueError(f'unknown divergence {divergence}')
-    elif callable(divergence):
-        return divergence
-    else:
-        raise ValueError(f'argument "divergence" not understood; use a str or a callable function')

 
-
-
[docs]def argmin_prevalence(loss, n_classes, method='optim_minimize'):
-    if method == 'optim_minimize':
-        return optim_minimize(loss, n_classes)
-    elif method == 'linear_search':
-        return linear_search(loss, n_classes)
-    elif method == 'ternary_search':
-        raise NotImplementedError()
-    else:
-        raise NotImplementedError()

-
-
-
[docs]def optim_minimize(loss, n_classes):
+

+[docs]
+def antagonistic_prevalence(p, strength=1):
     """
-    Searches for the optimal prevalence values, i.e., an `n_classes`-dimensional vector of the (`n_classes`-1)-simplex
-    that yields the smallest lost. This optimization is carried out by means of a constrained search using scipy's
-    SLSQP routine.
+    Reflects a prevalence vector in ILR space and maps it back to the simplex.
 
-    :param loss: (callable) the function to minimize
-    :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector
-    :return: (ndarray) the best prevalence vector found
+    :param p: array-like prevalence vector
+    :param strength: reflection strength in ILR space
+    :return: prevalence vector in the simplex
     """
-    from scipy import optimize
-
-    # the initial point is set as the uniform distribution
-    uniform_distribution = np.full(fill_value=1 / n_classes, shape=(n_classes,))
-
-    # solutions are bounded to those contained in the unit-simplex
-    bounds = tuple((0, 1) for _ in range(n_classes))  # values in [0,1]
-    constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)})  # values summing up to 1
-    r = optimize.minimize(loss, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints)
-    return r.x

+    ilr = ILRtransformation()
+    z = ilr(p)
+    z_ant = -strength * z
+    return ilr.inverse(z_ant)

 
 
-
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/_kdey.html b/docs/build/html/_modules/quapy/method/_kdey.html index 4e96e56..106a204 100644 --- a/docs/build/html/_modules/quapy/method/_kdey.html +++ b/docs/build/html/_modules/quapy/method/_kdey.html @@ -1,131 +1,455 @@ + - - - - - quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method._kdey — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.method._kdey

-from typing import Union
-import numpy as np
-from sklearn.base import BaseEstimator
-from sklearn.neighbors import KernelDensity
+import numpy as np
+from numbers import Real
+from sklearn.base import BaseEstimator
+from sklearn.neighbors import KernelDensity
 
-import quapy as qp
-from quapy.data import LabelledCollection
-from quapy.method.aggregative import AggregativeSoftQuantifier
-import quapy.functional as F
-
-from sklearn.metrics.pairwise import rbf_kernel
+import quapy as qp
+from quapy.method._helper import _labels_to_indices
+from quapy.method.aggregative import AggregativeSoftQuantifier
+import quapy.functional as F
+from scipy.special import logsumexp
+from sklearn.metrics.pairwise import rbf_kernel
 
 
-
[docs]class KDEBase:
+

+[docs]
+class KDEBase:
     """
     Common ancestor for KDE-based methods. Implements some common routines.
     """
 
     BANDWIDTH_METHOD = ['scott', 'silverman']
+    KERNELS = ['gaussian', 'aitchison', 'ilr']
 
     @classmethod
-    def _check_bandwidth(cls, bandwidth):
+    def _check_bandwidth(cls, bandwidth, kernel):
         """
         Checks that the bandwidth parameter is correct
 
         :param bandwidth: either a string (see BANDWIDTH_METHOD) or a float
-        :return: nothing, but raises an exception for invalid values
+        :return: the bandwidth if the check is passed, or raises an exception for invalid values
         """
-        assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, float), \
+        assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, Real), \
             f'invalid bandwidth, valid ones are {KDEBase.BANDWIDTH_METHOD} or float values'
-        if isinstance(bandwidth, float):
-            assert 0 < bandwidth < 1,  "the bandwith for KDEy should be in (0,1), since this method models the unit simplex"
+        if isinstance(bandwidth, Real):
+            bandwidth = float(bandwidth)
+        return bandwidth
 
-
[docs]    def get_kde_function(self, X, bandwidth):
+    @classmethod
+    def _check_kernel(cls, kernel):
+        assert kernel in KDEBase.KERNELS, f'unknown {kernel=}'
+        return kernel
+
+

+[docs]
+    def get_kde_function(self, X, bandwidth, kernel):
         """
         Wraps the KDE function from scikit-learn.
 
         :param X: data for which the density function is to be estimated
         :param bandwidth: the bandwidth of the kernel
+        :param kernel: the kernel family
         :return: a scikit-learn's KernelDensity object
         """
+        X = self.transform_posteriors(X, kernel)
+        bandwidth = self.effective_bandwidth(bandwidth, kernel)
         return KernelDensity(bandwidth=bandwidth).fit(X)

 
-
[docs]    def pdf(self, kde, X):
+
+

+[docs]
+    def pdf(self, kde, X, kernel, log_densities=False):
         """
         Wraps the density evalution of scikit-learn's KDE. Scikit-learn returns log-scores (s), so this
         function returns :math:`e^{s}`
 
         :param kde: a previously fit KDE function
         :param X: the data for which the density is to be estimated
+        :param kernel: the kernel family
         :return: np.ndarray with the densities
         """
-        return np.exp(kde.score_samples(X))

+        X = self.transform_posteriors(X, kernel)
+        log_density = kde.score_samples(X)
+        if log_densities:
+            return log_density
+        return np.exp(log_density)

 
-
[docs]    def get_mixture_components(self, X, y, n_classes, bandwidth):
+
+

+[docs]
+    def get_mixture_components(self, X, y, classes, bandwidth, kernel):
         """
         Returns an array containing the mixture components, i.e., the KDE functions for each class.
 
@@ -133,17 +457,76 @@
         :param y: the class labels
         :param n_classes: integer, the number of classes
         :param bandwidth: float, the bandwidth of the kernel
+        :param kernel: the kernel family
         :return: a list of KernelDensity objects, each fitted with the corresponding class-specific covariates
         """
-        return [self.get_kde_function(X[y == cat], bandwidth) for cat in range(n_classes)]

+        class_cond_X = []
+        for cat in classes:
+            selX = X[y==cat]
+            if selX.size==0:
+                raise ValueError(f'empty class {cat}')
+            class_cond_X.append(selX)
+        return [self.get_kde_function(X_cond_yi, bandwidth, kernel) for X_cond_yi in class_cond_X]

+
+
+

+[docs]
+    def transform_posteriors(self, X, kernel):
+        if kernel in {'aitchison', 'ilr'}:
+            X = self.shrink_posteriors(X)
+        if kernel == 'aitchison':
+            return self.clr_transform(X)
+        if kernel == 'ilr':
+            return self.ilr_transform(X)
+        return X

+
+
+

+[docs]
+    def shrink_posteriors(self, X):
+        shrinkage = getattr(self, 'shrinkage', 0.0)
+        if shrinkage <= 0:
+            return X
+        X = np.asarray(X)
+        n_classes = X.shape[-1]
+        uniform = np.full(n_classes, 1.0 / n_classes, dtype=X.dtype)
+        return (1.0 - shrinkage) * X + shrinkage * uniform

+
+
+

+[docs]
+    def effective_bandwidth(self, bandwidth, kernel):
+        shrinkage = getattr(self, 'shrinkage', 0.0)
+        if shrinkage > 0 and kernel in {'aitchison', 'ilr'} and isinstance(bandwidth, Real):
+            return (1.0 - shrinkage) * float(bandwidth)
+        return bandwidth

+
+
+

+[docs]
+    def clr_transform(self, X):
+        if not hasattr(self, 'clr'):
+            self.clr = F.CLRtransformation()
+        return self.clr(X)

+
+
+

+[docs]
+    def ilr_transform(self, X):
+        if not hasattr(self, 'ilr'):
+            self.ilr = F.ILRtransformation()
+        return self.ilr(X)

+

 
 
 
-
[docs]class KDEyML(AggregativeSoftQuantifier, KDEBase):
+

+[docs]
+class KDEyML(AggregativeSoftQuantifier, KDEBase):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Kullback-Leibler divergence (KLD) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors show that minimizing the distribution mathing criterion for KLD is akin to performing
     maximum likelihood (ML).
 
@@ -165,32 +548,49 @@
 
     which corresponds to the maximum likelihood estimate.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
+    :param kernel: kernel of KDE, valid ones are in KDEBase.KERNELS
+    :param shrinkage: amount of shrinkage towards the uniform distribution to apply before
+        Aitchison/ILR transformations. Must be in ``[0,1)``.
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, bandwidth=0.1, n_jobs=None, random_state=None):
-        self._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1,
+                 kernel='gaussian', shrinkage=0.0, random_state=None):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel)
+        self.kernel = self._check_kernel(kernel)
+        assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)'
+        assert self.kernel != 'gaussian' or shrinkage == 0, \
+            'shrinkage is only supported for Aitchison/ILR kernels'
+        self.shrinkage = float(shrinkage)
         self.random_state=random_state
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        self.mix_densities = self.get_mixture_components(*classif_predictions.Xy, data.n_classes, self.bandwidth)
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self.mix_densities = self.get_mixture_components(
+            classif_predictions,
+            labels,
+            self.classes_,
+            self.bandwidth,
+            self.kernel,
+        )
         return self

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         """
         Searches for the mixture model parameter (the sought prevalence values) that maximizes the likelihood
         of the data (i.e., that minimizes the negative log-likelihood)
@@ -198,24 +598,39 @@
         :param posteriors: instances in the sample converted into posterior probabilities
         :return: a vector of class prevalence estimates
         """
-        np.random.RandomState(self.random_state)
-        epsilon = 1e-10
-        n_classes = len(self.mix_densities)
-        test_densities = [self.pdf(kde_i, posteriors) for kde_i in self.mix_densities]
+        with qp.util.temp_seed(self.random_state):
+            epsilon = 1e-12
+            n_classes = len(self.mix_densities)
+            if (self.kernel != 'gaussian' and n_classes >= 20) or n_classes >= 30:
+                test_log_densities = [
+                    self.pdf(kde_i, posteriors, self.kernel, log_densities=True)
+                    for kde_i in self.mix_densities
+                ]
 
-        def neg_loglikelihood(prev):
-            test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities))
-            test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
-            return  -np.sum(test_loglikelihood)
+                def neg_loglikelihood(prev):
+                    prev = qp.error.smooth(prev, eps=epsilon)
+                    test_loglikelihood = logsumexp(np.log(prev)[:, None] + test_log_densities, axis=0)
+                    return -np.sum(test_loglikelihood)
+            else:
+                test_densities = [self.pdf(kde_i, posteriors, self.kernel) for kde_i in self.mix_densities]
 
-        return F.optim_minimize(neg_loglikelihood, n_classes)

+                def neg_loglikelihood(prev):
+                    test_mixture_likelihood = prev @ test_densities
+                    test_loglikelihood = np.log(test_mixture_likelihood + epsilon)
+                    return -np.sum(test_loglikelihood)
+
+            return F.optim_minimize(neg_loglikelihood, n_classes)

+

 
 
-
[docs]class KDEyHD(AggregativeSoftQuantifier, KDEBase):
+
+

+[docs]
+class KDEyHD(AggregativeSoftQuantifier, KDEBase):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the squared Hellinger Disntace (HD) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors proposed a Monte Carlo approach for minimizing the divergence.
 
     The distribution matching optimization problem comes down to solving:
@@ -242,53 +657,59 @@
     where the datapoints (trials) :math:`x_1,\\ldots,x_t\\sim_{\\mathrm{iid}} r` with :math:`r`  the
     uniform distribution.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param montecarlo_trials: number of Monte Carlo trials (default 10000)
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, divergence: str='HD',
-                 bandwidth=0.1, n_jobs=None, random_state=None, montecarlo_trials=10000):
-        
-        self._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, divergence: str='HD',
+                 bandwidth=0.1, random_state=None, montecarlo_trials=10000):
+
+        super().__init__(classifier, fit_classifier, val_split)
         self.divergence = divergence
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian')
         self.random_state=random_state
         self.montecarlo_trials = montecarlo_trials
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        self.mix_densities = self.get_mixture_components(*classif_predictions.Xy, data.n_classes, self.bandwidth)
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self.mix_densities = self.get_mixture_components(
+            classif_predictions, labels, self.classes_, self.bandwidth, 'gaussian'
+        )
 
         N = self.montecarlo_trials
         rs = self.random_state
-        n = data.n_classes
+        n = len(self.classes_)
         self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.mix_densities])
-        self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.mix_densities])
+        self.reference_classwise_densities = np.asarray(
+            [self.pdf(kde_j, self.reference_samples, 'gaussian') for kde_j in self.mix_densities]
+        )
         self.reference_density = np.mean(self.reference_classwise_densities, axis=0)  # equiv. to (uniform @ self.reference_classwise_densities)
 
         return self

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         # we retain all n*N examples (sampled from a mixture with uniform parameter), and then
         # apply importance sampling (IS). In this version we compute D(p_alpha||q) with IS
         n_classes = len(self.mix_densities)
 
-        test_kde = self.get_kde_function(posteriors, self.bandwidth)
-        test_densities = self.pdf(test_kde, self.reference_samples)
+        test_kde = self.get_kde_function(posteriors, self.bandwidth, 'gaussian')
+        test_densities = self.pdf(test_kde, self.reference_samples, 'gaussian')
 
-        def f_squared_hellinger(u):
+        def f_squared_hellinger(u):
             return (np.sqrt(u)-1)**2
         
         # todo: this will fail when self.divergence is a callable, and is not the right place to do it anyway
@@ -304,19 +725,23 @@
         p_class = self.reference_classwise_densities + epsilon
         fracs = p_class/qs
 
-        def divergence(prev):
+        def divergence(prev):
             # ps / qs = (prev @ p_class) / qs = prev @ (p_class / qs) = prev @ fracs
             ps_div_qs = prev @ fracs
             return np.mean( f(ps_div_qs) * iw )
 
-        return F.optim_minimize(divergence, n_classes)

+        return F.optim_minimize(divergence, n_classes)

+

 
 
-
[docs]class KDEyCS(AggregativeSoftQuantifier):
+
+

+[docs]
+class KDEyCS(AggregativeSoftQuantifier):
     """
     Kernel Density Estimation model for quantification (KDEy) relying on the Cauchy-Schwarz divergence (CS) as
     the divergence measure to be minimized. This method was first proposed in the paper
-    `Kernel Density Estimation for Multiclass Quantification <https://arxiv.org/abs/2401.00490>`_, in which
+    `Kernel Density Estimation for Multiclass Quantification <https://link.springer.com/article/10.1007/s10994-024-06726-5>`_ (`arXiv <https://arxiv.org/abs/2401.00490>`_), in which
     the authors proposed a Monte Carlo approach for minimizing the divergence.
 
     The distribution matching optimization problem comes down to solving:
@@ -337,26 +762,25 @@
 
     The authors showed that this distribution matching admits a closed-form solution
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     :param bandwidth: float, the bandwidth of the Kernel
-    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=10, bandwidth=0.1, n_jobs=None):
-        KDEBase._check_bandwidth(bandwidth)
-        self.classifier = classifier
-        self.val_split = val_split
-        self.bandwidth = bandwidth
-        self.n_jobs = n_jobs
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian')
 
-
[docs]    def gram_matrix_mix_sum(self, X, Y=None):
+

+[docs]
+    def gram_matrix_mix_sum(self, X, Y=None):
         # this adapts the output of the rbf_kernel function (pairwise evaluations of Gaussian kernels k(x,y))
         # to contain pairwise evaluations of N(x|mu,Sigma1+Sigma2) with mu=y and Sigma1 and Sigma2 are 
         # two "scalar matrices" (h^2)*I each, so Sigma1+Sigma2 has scalar 2(h^2) (h is the bandwidth)
@@ -368,17 +792,18 @@
         gram = norm_factor * rbf_kernel(X, Y, gamma=gamma)
         return gram.sum()

 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
 
-        P, y = classif_predictions.Xy
-        n = data.n_classes
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
 
-        assert all(sorted(np.unique(y)) == np.arange(n)), \
-            'label name gaps not allowed in current implementation'
+        P, y = classif_predictions, labels
+        n = len(self.classes_)
+        y = _labels_to_indices(y, self.classes_)
 
         # counts_inv keeps track of the relative weight of each datapoint within its class
         # (i.e., the weight in its KDE model)
-        counts_inv = 1 / (data.counts())
+        counts_inv = 1 / (F.counts_from_labels(y, classes=np.arange(n)))
 
         # tr_tr_sums corresponds to symbol \overline{B} in the paper
         tr_tr_sums = np.zeros(shape=(n,n), dtype=float)
@@ -399,7 +824,10 @@
         return self

 
 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         Ptr = self.Ptr
         Pte = posteriors
         y = self.ytr
@@ -419,44 +847,89 @@
         for i in range(n):
             tr_te_sums[i] = self.gram_matrix_mix_sum(Ptr[y==i], Pte) 
 
-        def divergence(alpha):
+        def divergence(alpha):
             # called \overline{r} in the paper
             alpha_ratio = alpha * self.counts_inv
 
-            # recal that tr_te_sums already accounts for the constant terms (1/Li)*(1/M)
+            # recall that tr_te_sums already accounts for the constant terms (1/Li)*(1/M)
             partA = -np.log((alpha_ratio @ tr_te_sums) * Minv)
             partB = 0.5 * np.log(alpha_ratio @ tr_tr_sums @ alpha_ratio)
             return partA + partB #+ partC
 
-        return F.optim_minimize(divergence, n)

+        return F.optim_minimize(divergence, n)

+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/_neural.html b/docs/build/html/_modules/quapy/method/_neural.html index 706a7cc..e38d5eb 100644 --- a/docs/build/html/_modules/quapy/method/_neural.html +++ b/docs/build/html/_modules/quapy/method/_neural.html @@ -1,91 +1,393 @@ + - - - - - quapy.method._neural — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method._neural — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.method._neural

-import os
-from pathlib import Path
-import random
+import logging
+import os
+from pathlib import Path
+import random
 
-import torch
-from torch.nn import MSELoss
-from torch.nn.functional import relu
+import torch
+from torch.nn import MSELoss
+from torch.nn.functional import relu
 
-from quapy.protocol import UPP
-from quapy.method.aggregative import *
-from quapy.util import EarlyStop
-from tqdm import tqdm
+from quapy.protocol import UPP
+from quapy.method.aggregative import *
+from quapy.util import EarlyStop
+from tqdm import tqdm
 
 
-
[docs]class QuaNetTrainer(BaseQuantifier):
+

+[docs]
+class QuaNetTrainer(BaseQuantifier):
     """
     Implementation of `QuaNet <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_, a neural network for
     quantification. This implementation uses `PyTorch <https://pytorch.org/>`_ and can take advantage of GPU
@@ -100,7 +402,7 @@
     >>> # use samples of 100 elements
     >>> qp.environ['SAMPLE_SIZE'] = 100
     >>>
-    >>> # load the kindle dataset as text, and convert words to numerical indexes
+    >>> # load the Kindle dataset as text, and convert words to numerical indexes
     >>> dataset = qp.datasets.fetch_reviews('kindle', pickle=True)
     >>> qp.train.preprocessing.index(dataset, min_df=5, inplace=True)
     >>>
@@ -110,12 +412,14 @@
     >>>
     >>> # train QuaNet (QuaNet is an alias to QuaNetTrainer)
     >>> model = QuaNet(classifier, qp.environ['SAMPLE_SIZE'], device='cuda')
-    >>> model.fit(dataset.training)
-    >>> estim_prevalence = model.quantify(dataset.test.instances)
+    >>> model.fit(*dataset.training.Xy)
+    >>> estim_prevalence = model.predict(dataset.test.instances)
 
     :param classifier: an object implementing `fit` (i.e., that can be trained on labelled data),
         `predict_proba` (i.e., that can generate posterior probabilities of unlabelled examples) and
         `transform` (i.e., that can generate embedded representations of the unlabelled instances).
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
     :param sample_size: integer, the sample size; default is None, meaning that the sample size should be
         taken from qp.environ["SAMPLE_SIZE"]
     :param n_epochs: integer, maximum number of training epochs
@@ -135,8 +439,9 @@
     :param device: string, indicate "cpu" or "cuda"
     """
 
-    def __init__(self,
+    def __init__(self,
                  classifier,
+                 fit_classifier=True,
                  sample_size=None,
                  n_epochs=100,
                  tr_iter_per_poch=500,
@@ -159,6 +464,7 @@
             f'the classifier {classifier.__class__.__name__} does not seem to be able to produce posterior probabilities ' \
                 f'since it does not implement the method "predict_proba"'
         self.classifier = classifier
+        self.fit_classifier = fit_classifier
         self.sample_size = qp._get_sample_size(sample_size)
         self.n_epochs = n_epochs
         self.tr_iter = tr_iter_per_poch
@@ -184,20 +490,23 @@
         self.__check_params_colision(self.quanet_params, self.classifier.get_params())
         self._classes_ = None
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True):
+

+[docs]
+    def fit(self, X, y):
         """
         Trains QuaNet.
 
-        :param data: the training data on which to train QuaNet. If `fit_classifier=True`, the data will be split in
+        :param X: the training instances on which to train QuaNet. If `fit_classifier=True`, the data will be split in
             40/40/20 for training the classifier, training QuaNet, and validating QuaNet, respectively. If
             `fit_classifier=False`, the data will be split in 66/34 for training QuaNet and validating it, respectively.
-        :param fit_classifier: if True, trains the classifier on a split containing 40% of the data
+        :param y: the labels of X
         :return: self
         """
+        data = LabelledCollection(X, y)
         self._classes_ = data.classes_
         os.makedirs(self.checkpointdir, exist_ok=True)
 
-        if fit_classifier:
+        if self.fit_classifier:
             classifier_data, unused_data = data.split_stratified(0.4)
             train_data, valid_data = unused_data.split_stratified(0.66)  # 0.66 split of 60% makes 40% and 20%
             self.classifier.fit(*classifier_data.Xy)
@@ -217,13 +526,13 @@
         train_data_embed = LabelledCollection(self.classifier.transform(train_data.instances), train_data.labels, self._classes_)
 
         self.quantifiers = {
-            'cc': CC(self.classifier).fit(None, fit_classifier=False),
-            'acc': ACC(self.classifier).fit(None, fit_classifier=False, val_split=valid_data),
-            'pcc': PCC(self.classifier).fit(None, fit_classifier=False),
-            'pacc': PACC(self.classifier).fit(None, fit_classifier=False, val_split=valid_data),
+            'cc': CC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'acc': ACC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'pcc': PCC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
+            'pacc': PACC(self.classifier, fit_classifier=False).fit(*valid_data.Xy),
         }
         if classifier_data is not None:
-            self.quantifiers['emq'] = EMQ(self.classifier).fit(classifier_data, fit_classifier=False)
+            self.quantifiers['emq'] = EMQ(self.classifier, fit_classifier=False).fit(*valid_data.Xy)
 
         self.status = {
             'tr-loss': -1,
@@ -241,7 +550,7 @@
             order_by=0 if data.binary else None,
             **self.quanet_params
         ).to(self.device)
-        print(self.quanet)
+        logging.getLogger(__name__).debug(self.quanet)
 
         self.optim = torch.optim.Adam(self.quanet.parameters(), lr=self.lr)
         early_stop = EarlyStop(self.patience, lower_is_better=True)
@@ -256,14 +565,16 @@
             if early_stop.IMPROVED:
                 torch.save(self.quanet.state_dict(), checkpoint)
             elif early_stop.STOP:
-                print(f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
-                      f'for epoch {early_stop.best_epoch}')
+                logging.getLogger(__name__).info(
+                    f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
+                    f'for epoch {early_stop.best_epoch}')
                 self.quanet.load_state_dict(torch.load(checkpoint))
                 break
 
         return self

 
-    def _get_aggregative_estims(self, posteriors):
+
+    def _get_aggregative_estims(self, posteriors):
         label_predictions = np.argmax(posteriors, axis=-1)
         prevs_estim = []
         for quantifier in self.quantifiers.values():
@@ -274,9 +585,11 @@
 
         return prevs_estim
 
-
[docs]    def quantify(self, instances):
-        posteriors = self.classifier.predict_proba(instances)
-        embeddings = self.classifier.transform(instances)
+

+[docs]
+    def predict(self, X):
+        posteriors = self.classifier.predict_proba(X)
+        embeddings = self.classifier.transform(X)
         quant_estims = self._get_aggregative_estims(posteriors)
         self.quanet.eval()
         with torch.no_grad():
@@ -286,7 +599,8 @@
             prevalence = prevalence.numpy().flatten()
         return prevalence

 
-    def _epoch(self, data: LabelledCollection, posteriors, iterations, epoch, early_stop, train):
+
+    def _epoch(self, data: LabelledCollection, posteriors, iterations, epoch, early_stop, train):
         mse_loss = MSELoss()
 
         self.quanet.train(mode=train)
@@ -336,12 +650,17 @@
                                      f'val-mseloss={self.status["va-loss"]:.5f} val-maeloss={self.status["va-mae"]:.5f} '
                                      f'patience={early_stop.patience}/{early_stop.PATIENCE_LIMIT}')
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         classifier_params = self.classifier.get_params()
         classifier_params = {'classifier__'+k:v for k,v in classifier_params.items()}
         return {**classifier_params, **self.quanet_params}

 
-
[docs]    def set_params(self, **parameters):
+
+

+[docs]
+    def set_params(self, **parameters):
         learner_params = {}
         for key, val in parameters.items():
             if key in self.quanet_params:
@@ -352,7 +671,8 @@
                 raise ValueError('unknown parameter ', key)
         self.classifier.set_params(**learner_params)

 
-    def __check_params_colision(self, quanet_params, learner_params):
+
+    def __check_params_colision(self, quanet_params, learner_params):
         quanet_keys = set(quanet_params.keys())
         learner_keys = set(learner_params.keys())
         intersection = quanet_keys.intersection(learner_keys)
@@ -360,25 +680,34 @@
             raise ValueError(f'the use of parameters {intersection} is ambiguous sine those can refer to '
                              f'the parameters of QuaNet or the learner {self.classifier.__class__.__name__}')
 
-
[docs]    def clean_checkpoint(self):
+

+[docs]
+    def clean_checkpoint(self):
         """
         Removes the checkpoint
         """
         os.remove(self.checkpoint)

 
-
[docs]    def clean_checkpoint_dir(self):
+
+

+[docs]
+    def clean_checkpoint_dir(self):
         """
         Removes anything contained in the checkpoint directory
         """
-        import shutil
+        import shutil
         shutil.rmtree(self.checkpointdir, ignore_errors=True)

 
+
     @property
-    def classes_(self):
+    def classes_(self):
         return self._classes_

 
 
-
[docs]def mae_loss(output, target):
+
+

+[docs]
+def mae_loss(output, target):
     """
     Torch-like wrapper for the Mean Absolute Error
 
@@ -389,7 +718,10 @@
     return torch.mean(torch.abs(output - target))

 
 
-
[docs]class QuaNetModule(torch.nn.Module):
+
+

+[docs]
+class QuaNetModule(torch.nn.Module):
     """
     Implements the `QuaNet <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_ forward pass.
     See :class:`QuaNetTrainer` for training QuaNet.
@@ -406,7 +738,7 @@
     :param order_by: integer, class for which the document embeddings are to be sorted
     """
 
-    def __init__(self,
+    def __init__(self,
                  doc_embedding_size,
                  n_classes,
                  stats_size,
@@ -441,10 +773,10 @@
         self.output = torch.nn.Linear(prev_size, n_classes)
 
     @property
-    def device(self):
+    def device(self):
         return torch.device('cuda') if next(self.parameters()).is_cuda else torch.device('cpu')
 
-    def _init_hidden(self):
+    def _init_hidden(self):
         directions = 2 if self.bidirectional else 1
         var_hidden = torch.zeros(self.nlayers * directions, 1, self.hidden_size)
         var_cell = torch.zeros(self.nlayers * directions, 1, self.hidden_size)
@@ -452,7 +784,9 @@
             var_hidden, var_cell = var_hidden.cuda(), var_cell.cuda()
         return var_hidden, var_cell
 
-
[docs]    def forward(self, doc_embeddings, doc_posteriors, statistics):
+

+[docs]
+    def forward(self, doc_embeddings, doc_posteriors, statistics):
         device = self.device
         doc_embeddings = torch.as_tensor(doc_embeddings, dtype=torch.float, device=device)
         doc_posteriors = torch.as_tensor(doc_posteriors, dtype=torch.float, device=device)
@@ -482,7 +816,9 @@
         logits = self.output(abstracted).view(1, -1)
         prevalence = torch.softmax(logits, -1)
 
-        return prevalence

+        return prevalence

+

+
 
 
 
@@ -490,31 +826,75 @@
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.19.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/_threshold_optim.html b/docs/build/html/_modules/quapy/method/_threshold_optim.html index 486aa61..74ddfb0 100644 --- a/docs/build/html/_modules/quapy/method/_threshold_optim.html +++ b/docs/build/html/_modules/quapy/method/_threshold_optim.html @@ -1,87 +1,388 @@ + - - - - - quapy.method._threshold_optim — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method._threshold_optim — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.method._threshold_optim

-from abc import abstractmethod
+from abc import abstractmethod
 
-import numpy as np
-from sklearn.base import BaseEstimator
-import quapy as qp
-import quapy.functional as F
-from quapy.data import LabelledCollection
-from quapy.method.aggregative import BinaryAggregativeQuantifier
+import numpy as np
+from sklearn.base import BaseEstimator
+import quapy as qp
+import quapy.functional as F
+from quapy.data import LabelledCollection
+from quapy.method.aggregative import BinaryAggregativeQuantifier
 
 
-
[docs]class ThresholdOptimization(BinaryAggregativeQuantifier):
+

+[docs]
+class ThresholdOptimization(BinaryAggregativeQuantifier):
     """
     Abstract class of Threshold Optimization variants for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -91,22 +392,29 @@
     that would allow for more true positives and many more false positives, on the grounds this
     would deliver larger denominators.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=None, n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    @abstractmethod
-    def condition(self, tpr, fpr) -> float:
+

+[docs]
+    @abstractmethod
+    def condition(self, tpr, fpr) -> float:
         """
         Implements the criterion according to which the threshold should be selected.
         This function should return the (float) score to be minimized.
@@ -117,7 +425,10 @@
         """
         ...

 
-
[docs]    def discard(self, tpr, fpr) -> bool:
+
+

+[docs]
+    def discard(self, tpr, fpr) -> bool:
         """
         Indicates whether a combination of tpr and fpr should be discarded
 
@@ -128,7 +439,8 @@
         return (tpr - fpr) == 0

 
 
-    def _eval_candidate_thresholds(self, decision_scores, y):
+
+    def _eval_candidate_thresholds(self, decision_scores, y):
         """
         Seeks for the best `tpr` and `fpr` according to the score obtained at different
         decision thresholds. The scoring function is implemented in function `_condition`.
@@ -163,7 +475,9 @@
 
         return candidates
 
-
[docs]    def aggregate_with_threshold(self, classif_predictions, tprs, fprs, thresholds):
+

+[docs]
+    def aggregate_with_threshold(self, classif_predictions, tprs, fprs, thresholds):
         # This function performs the adjusted count for given tpr, fpr, and threshold.
         # Note that, due to broadcasting, tprs, fprs, and thresholds could be arrays of length > 1
         prevs_estims = np.mean(classif_predictions[:, None] >= thresholds, axis=0)
@@ -171,35 +485,45 @@
         prevs_estims = F.as_binary_prevalence(prevs_estims, clip_if_necessary=True)
         return prevs_estims.squeeze()

 
-    def _compute_table(self, y, y_):
+
+    def _compute_table(self, y, y_):
         TP = np.logical_and(y == y_, y == self.pos_label).sum()
         FP = np.logical_and(y != y_, y == self.neg_label).sum()
         FN = np.logical_and(y != y_, y == self.pos_label).sum()
         TN = np.logical_and(y == y_, y == self.neg_label).sum()
         return TP, FP, FN, TN
 
-    def _compute_tpr(self, TP, FP):
-        if TP + FP == 0:
+    def _compute_tpr(self, TP, FN):
+        if TP + FN == 0:
             return 1
-        return TP / (TP + FP)
+        return TP / (TP + FN)
 
-    def _compute_fpr(self, FP, TN):
+    def _compute_fpr(self, FP, TN):
         if FP + TN == 0:
             return 0
         return FP / (FP + TN)
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        decision_scores, y = classif_predictions.Xy
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        decision_scores, y = classif_predictions, labels
         # the standard behavior is to keep the best threshold only
         self.tpr, self.fpr, self.threshold = self._eval_candidate_thresholds(decision_scores, y)[0]
         return self

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
         # the standard behavior is to compute the adjusted count using the best threshold found
-        return self.aggregate_with_threshold(classif_predictions, self.tpr, self.fpr, self.threshold)

+        return self.aggregate_with_threshold(classif_predictions, self.tpr, self.fpr, self.threshold)

+

 
 
-
[docs]class T50(ThresholdOptimization):
+
+

+[docs]
+class T50(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -207,23 +531,34 @@
     for the threshold that makes `tpr` closest to 0.5.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
-        return abs(tpr - 0.5)

+

+[docs]
+    def condition(self, tpr, fpr) -> float:
+        return abs(tpr - 0.5)

+

 
 
-
[docs]class MAX(ThresholdOptimization):
+
+

+[docs]
+class MAX(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -231,24 +566,33 @@
     for the threshold that maximizes `tpr-fpr`.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
+

+[docs]
+    def condition(self, tpr, fpr) -> float:
         # MAX strives to maximize (tpr - fpr), which is equivalent to minimize (fpr - tpr)
-        return (fpr - tpr)

+        return (fpr - tpr)

+

 
 
-
[docs]class X(ThresholdOptimization):
+
+

+[docs]
+class X(ThresholdOptimization):
     """
     Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -256,23 +600,32 @@
     for the threshold that yields `tpr=1-fpr`.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
-        return abs(1 - (tpr + fpr))

+

+[docs]
+    def condition(self, tpr, fpr) -> float:
+        return abs(1 - (tpr + fpr))

+

 
 
-
[docs]class MS(ThresholdOptimization):
+
+

+[docs]
+class MS(ThresholdOptimization):
     """
     Median Sweep. Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -280,22 +633,30 @@
     class prevalence estimates for all decision thresholds and returns the median of them all.
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
 
-
[docs]    def condition(self, tpr, fpr) -> float:
+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def condition(self, tpr, fpr) -> float:
         return 1

 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        decision_scores, y = classif_predictions.Xy
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        decision_scores, y = classif_predictions, labels
         # keeps all candidates
         tprs_fprs_thresholds = self._eval_candidate_thresholds(decision_scores, y)
         self.tprs = tprs_fprs_thresholds[:, 0]
@@ -303,14 +664,21 @@
         self.thresholds = tprs_fprs_thresholds[:, 2]
         return self

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
         prevalences = self.aggregate_with_threshold(classif_predictions, self.tprs, self.fprs, self.thresholds)
         if prevalences.ndim==2:
             prevalences = np.median(prevalences, axis=0)
-        return prevalences

+        return prevalences

+

 
 
-
[docs]class MS2(MS):
+
+

+[docs]
+class MS2(MS):
     """
     Median Sweep 2. Threshold Optimization variant for :class:`ACC` as proposed by
     `Forman 2006 <https://dl.acm.org/doi/abs/10.1145/1150402.1150423>`_ and
@@ -319,46 +687,97 @@
     which `tpr-fpr>0.25`
     The goal is to bring improved stability to the denominator of the adjustment.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set in which the
-        misclassification rates are to be estimated.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the misclassification rates should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        super().__init__(classifier, val_split)
 
-
[docs]    def discard(self, tpr, fpr) -> bool:
-        return (tpr-fpr) <= 0.25

+    def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def discard(self, tpr, fpr) -> bool:
+        return (tpr-fpr) <= 0.25

+

+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/aggregative.html b/docs/build/html/_modules/quapy/method/aggregative.html index 8311baa..a6aa441 100644 --- a/docs/build/html/_modules/quapy/method/aggregative.html +++ b/docs/build/html/_modules/quapy/method/aggregative.html @@ -1,100 +1,413 @@ + - - - - - quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method.aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ +

Source code for quapy.method.aggregative

+import warnings
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from typing import Callable, Literal, Union
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.exceptions import NotFittedError
+from sklearn.metrics import confusion_matrix
+from sklearn.model_selection import cross_val_predict, train_test_split
+from sklearn.utils.validation import check_is_fitted
+
+import quapy as qp
+import quapy.functional as F
+from quapy.functional import get_divergence
+from quapy.classification.svmperf import SVMperf
+from quapy.data import LabelledCollection
+from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric
+from quapy.method._energy import _EnergyDistanceCore
+from quapy.method._helper import (
+    _get_abstention_calibrators,
+    _get_cvxpy,
+    _rlls_check_mode,
+    _rlls_joint_distribution,
+    _rlls_predicted_marginal,
+    _rlls_compute_3deltaC,
+    _rlls_compute_weights,
+    _labels_to_indices,
+)
 
-import quapy as qp
-import quapy.functional as F
-from quapy.functional import get_divergence
-from quapy.classification.calibration import NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration
-from quapy.classification.svmperf import SVMperf
-from quapy.data import LabelledCollection
-from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric
 
 
 # Abstract classes
 # ------------------------------------
 
-
[docs]class AggregativeQuantifier(BaseQuantifier, ABC):
+

+[docs]
+class AggregativeQuantifier(BaseQuantifier, ABC):
     """
     Abstract class for quantification methods that base their estimations on the aggregation of classification
     results. Aggregative quantifiers implement a pipeline that consists of generating classification predictions
@@ -108,22 +421,82 @@
 
     The method :meth:`quantify` comes with a default implementation based on :meth:`classify`
     and :meth:`aggregate`.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple `(X,y)` defining the specific set of data to use for validation. Set to
+        None when the method does not require any validation data, in order to avoid that some portion of
+        the training data be wasted.
     """
 
-    val_split_ = None
+    def __init__(self,
+                 classifier: Union[None,BaseEstimator],
+                 fit_classifier:bool=True,
+                 val_split:Union[int,float,tuple,None]=5):
 
-    @property
-    def val_split(self):
-        return self.val_split_
+        self.classifier = qp._get_classifier(classifier)
+        self.fit_classifier = fit_classifier
+        self.val_split = val_split
 
-    @val_split.setter
-    def val_split(self, val_split):
-        if isinstance(val_split, LabelledCollection):
-            print('warning: setting val_split with a LabelledCollection will be inefficient in'
-                  'model selection. Rather pass the LabelledCollection at fit time')
-        self.val_split_ = val_split
+        # basic type checks
+        assert hasattr(self.classifier, 'fit'), \
+            f'the classifier does not implement "fit"'
 
-    def _check_init_parameters(self):
+        assert isinstance(fit_classifier, bool), \
+            f'unexpected type for {fit_classifier=}; must be True or False'
+
+        # val_split is indicated as a number of folds for cross-validation
+        if isinstance(val_split, int):
+            assert val_split > 1, \
+                (f'when {val_split=} is indicated as an integer, it represents the number of folds in a kFCV '
+                 f'and must thus be >1')
+            if val_split==5 and not fit_classifier:
+                warnings.warn(f'{val_split=} will be ignored when the classifier is already trained '
+                              f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} '
+                              f'to None to avoid this warning.')
+                self.val_split=None
+            if val_split!=5:
+                assert fit_classifier, (f'Parameter {val_split=} has been modified, but {fit_classifier=} '
+                                        f'indicates the classifier should not be retrained.')
+        # val_split is indicated as a fraction of validation instances
+        elif isinstance(val_split, float):
+            assert 0 < val_split < 1, \
+                (f'when {val_split=} is indicated as a float, it represents the fraction of training instances '
+                 f'to be used for validation, and must thus be in the range (0,1)')
+            assert fit_classifier, (f'when {val_split=} is indicated as a float (the fraction of training instances '
+                                    f'to be used for validation), the parameter {fit_classifier=} must be True')
+        # val_split is indicated as a validation collection (X,y)
+        elif isinstance(val_split, tuple):
+            assert len(val_split) == 2, \
+                (f'when {val_split=} is indicated as a tuple, it represents the collection (X,y) on which the '
+                 f'validation must be performed, but this seems to have different cardinality')
+        elif val_split is None:
+            pass
+        else:
+            raise ValueError(f'unexpected type for {val_split=}')
+
+        # classifier is fitted?
+        try:
+            check_is_fitted(self.classifier)
+            fitted = True
+        except NotFittedError:
+            fitted = False
+
+        # consistency checks: fit_classifier?
+        if self.fit_classifier:
+            if fitted:
+                raise RuntimeWarning(f'the classifier is already fitted, but {fit_classifier=} was requested')
+        else:
+            assert fitted, (f'{fit_classifier=} requires the classifier to be already trained, '
+                            f'but this does not seem to be')
+
+    def _check_init_parameters(self):
         """
         Implements any check to be performed in the parameters of the init method before undertaking
         the training of the quantifier. This is made as to allow for a quick execution stop when the
@@ -133,116 +506,103 @@
         """
         pass
 
-    def _check_non_empty_classes(self, data: LabelledCollection):
+    def _check_non_empty_classes(self, y):
         """
         Asserts all classes have positive instances.
 
-        :param data: LabelledCollection
+        :param labels: array-like of shape `(n_instances,)` with the label for each instance
+        :param classes: the class labels. This is needed in order to correctly compute the prevalence vector even when
+            some classes have no examples.
         :return: Nothing. May raise an exception.
         """
-        sample_prevs = data.prevalence()
-        empty_classes = np.argwhere(sample_prevs==0).flatten()
-        if len(empty_classes)>0:
-            empty_class_names = data.classes_[empty_classes]
+        sample_prevs = F.prevalence_from_labels(y, self.classes_)
+        empty_classes = np.argwhere(sample_prevs == 0).flatten()
+        if len(empty_classes) > 0:
+            empty_class_names = self.classes_[empty_classes]
             raise ValueError(f'classes {empty_class_names} have no training examples')
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True, val_split=None):
+

+[docs]
+    def fit(self, X, y):
         """
-        Trains the aggregative quantifier. This comes down to training a classifier and an aggregation function.
+        Trains the aggregative quantifier. This comes down to training a classifier (if requested) and an
+        aggregation function.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
-        :param fit_classifier: whether to train the learner (default is True). Set to False if the
-            learner has been trained outside the quantifier.
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         :return: self
         """
         self._check_init_parameters()
-        classif_predictions = self.classifier_fit_predict(data, fit_classifier, predict_on=val_split)
-        self.aggregation_fit(classif_predictions, data)
+        classif_predictions, labels = self.classifier_fit_predict(X, y)
+        self.aggregation_fit(classif_predictions, labels)
         return self

 
-
[docs]    def classifier_fit_predict(self, data: LabelledCollection, fit_classifier=True, predict_on=None):
+
+

+[docs]
+    def classifier_fit_predict(self, X, y):
         """
         Trains the classifier if requested (`fit_classifier=True`) and generate the necessary predictions to
         train the aggregation function.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
-        :param fit_classifier: whether to train the learner (default is True). Set to False if the
-            learner has been trained outside the quantifier.
-        :param predict_on: specifies the set on which predictions need to be issued. This parameter can
-            be specified as None (default) to indicate no prediction is needed; a float in (0, 1) to
-            indicate the proportion of instances to be used for predictions (the remainder is used for
-            training); an integer >1 to indicate that the predictions must be generated via k-fold
-            cross-validation, using this integer as k; or the data sample itself on which to generate
-            the predictions.
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         """
-        assert isinstance(fit_classifier, bool), 'unexpected type for "fit_classifier", must be boolean'
+        self._check_classifier(adapt_if_necessary=self.fit_classifier)
 
-        self._check_classifier(adapt_if_necessary=(self._classifier_method() == 'predict_proba'))
+        # self._check_non_empty_classes(y)
 
-        if fit_classifier:
-            self._check_non_empty_classes(data)
-
-        if predict_on is None:
-            predict_on = self.val_split
-
-        if predict_on is None:
-            if fit_classifier:
-                self.classifier.fit(*data.Xy)
-            predictions = None
-        elif isinstance(predict_on, float):
-            if fit_classifier:
-                if not (0. < predict_on < 1.):
-                    raise ValueError(f'proportion {predict_on=} out of range, must be in (0,1)')
-                train, val = data.split_stratified(train_prop=(1 - predict_on))
-                self.classifier.fit(*train.Xy)
-                predictions = LabelledCollection(self.classify(val.X), val.y, classes=data.classes_)
+        predictions, labels = None, None
+        if isinstance(self.val_split, int):
+            assert self.fit_classifier, f'{self.__class__}: unexpected value for {self.fit_classifier=}'
+            num_folds = self.val_split
+            n_jobs = self.n_jobs if hasattr(self, 'n_jobs') else qp._get_njobs(None)
+            predictions = cross_val_predict(
+                self.classifier, X, y, cv=num_folds, n_jobs=n_jobs, method=self._classifier_method()
+            )
+            labels = y
+            self.classifier.fit(X, y)
+        elif isinstance(self.val_split, float):
+            assert self.fit_classifier, f'unexpected value for {self.fit_classifier=}'
+            train_prop = 1. - self.val_split
+            Xtr, Xval, ytr, yval = train_test_split(X, y, train_size=train_prop, stratify=y)
+            self.classifier.fit(Xtr, ytr)
+            predictions = self.classify(Xval)
+            labels = yval
+        elif isinstance(self.val_split, tuple):
+            Xval, yval = self.val_split
+            if self.fit_classifier:
+                self.classifier.fit(X, y)
+            predictions = self.classify(Xval)
+            labels = yval
+        elif self.val_split is None:
+            if self.fit_classifier:
+                self.classifier.fit(X, y)
+                predictions, labels = None, None
             else:
-                raise ValueError(f'wrong type for predict_on: since fit_classifier=False, '
-                                 f'the set on which predictions have to be issued must be '
-                                 f'explicitly indicated')
-
-        elif isinstance(predict_on, LabelledCollection):
-            if fit_classifier:
-                self.classifier.fit(*data.Xy)
-            predictions = LabelledCollection(self.classify(predict_on.X), predict_on.y, classes=predict_on.classes_)
-
-        elif isinstance(predict_on, int):
-            if fit_classifier:
-                if predict_on <= 1:
-                    raise ValueError(f'invalid value {predict_on} in fit. '
-                                     f'Specify a integer >1 for kFCV estimation.')
-                else:
-                    n_jobs = self.n_jobs if hasattr(self, 'n_jobs') else qp._get_njobs(None)
-                    predictions = cross_val_predict(
-                        self.classifier, *data.Xy, cv=predict_on, n_jobs=n_jobs, method=self._classifier_method())
-                    predictions = LabelledCollection(predictions, data.y, classes=data.classes_)
-                    self.classifier.fit(*data.Xy)
-            else:
-                raise ValueError(f'wrong type for predict_on: since fit_classifier=False, '
-                                 f'the set on which predictions have to be issued must be '
-                                 f'explicitly indicated')
-
+                predictions, labels = self.classify(X), y
         else:
-            raise ValueError(
-                f'error: param "predict_on" ({type(predict_on)}) not understood; '
-                f'use either a float indicating the split proportion, or a '
-                f'tuple (X,y) indicating the validation partition')
+            raise ValueError(f'unexpected type for {self.val_split=}')
 
-        return predictions

+        return predictions, labels

 
-
[docs]    @abstractmethod
-    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+
+

+[docs]
+    @abstractmethod
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Trains the aggregation function.
 
-        :param classif_predictions: a LabelledCollection containing the label predictions issued
-            by the classifier
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
+        :param classif_predictions: array-like with the classification predictions
+            (whatever the method :meth:`classify` returns)
+        :param labels: array-like with the true labels associated to each classifier prediction
         """
         ...

 
+
     @property
-    def classifier(self):
+    def classifier(self):
         """
         Gives access to the classifier
 
@@ -251,7 +611,7 @@
         return self.classifier_
 
     @classifier.setter
-    def classifier(self, classifier):
+    def classifier(self, classifier):
         """
         Setter for the classifier
 
@@ -259,18 +619,21 @@
         """
         self.classifier_ = classifier
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
         Provides the label predictions for the given instances. The predictions should respect the format expected by
         :meth:`aggregate`, e.g., posterior probabilities for probabilistic quantifiers, or crisp predictions for
         non-probabilistic quantifiers. The default one is "decision_function".
 
-        :param instances: array-like of shape `(n_instances, n_features,)`
-        :return: np.ndarray of shape `(n_instances,)` with label predictions
+        :param X: array-like of shape `(n_samples, n_features)`, the data instances
+        :return: np.ndarray of shape `(n_instances,)` with classifier predictions
         """
-        return getattr(self.classifier, self._classifier_method())(instances)

+        return getattr(self.classifier, self._classifier_method())(X)

 
-    def _classifier_method(self):
+
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. The default one is "decision_function".
 
@@ -278,56 +641,65 @@
         """
         return 'decision_function'
 
-    def _check_classifier(self, adapt_if_necessary=False):
+    def _check_classifier(self, adapt_if_necessary=False):
         """
         Guarantees that the underlying classifier implements the method required for issuing predictions, i.e.,
         the method indicated by the :meth:`_classifier_method`
 
-        :param adapt_if_necessary: if True, the method will try to comply with the required specifications
+        :param adapt_if_necessary: unused unless overridden
         """
         assert hasattr(self.classifier, self._classifier_method()), \
             f"the method does not implement the required {self._classifier_method()} method"
 
-
[docs]    def quantify(self, instances):
+

+[docs]
+    def predict(self, X):
         """
         Generate class prevalence estimates for the sample's instances by aggregating the label predictions generated
         by the classifier.
 
-        :param instances: array-like
+        :param X: array-like of shape `(n_samples, n_features)`, the data instances
         :return: `np.ndarray` of shape `(n_classes)` with class prevalence estimates.
         """
-        classif_predictions = self.classify(instances)
+        classif_predictions = self.classify(X)
         return self.aggregate(classif_predictions)

 
-
[docs]    @abstractmethod
-    def aggregate(self, classif_predictions: np.ndarray):
-        """
-        Implements the aggregation of label predictions.
 
-        :param classif_predictions: `np.ndarray` of label predictions
+

+[docs]
+    @abstractmethod
+    def aggregate(self, classif_predictions: np.ndarray):
+        """
+        Implements the aggregation of the classifier predictions.
+
+        :param classif_predictions: `np.ndarray` of classifier predictions
         :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
         """
         ...

 
+
     @property
-    def classes_(self):
+    def classes_(self):
         """
         Class labels, in the same order in which class prevalence values are to be computed.
         This default implementation actually returns the class labels of the learner.
 
-        :return: array-like
+        :return: array-like, the class labels
         """
         return self.classifier.classes_

 
 
-
[docs]class AggregativeCrispQuantifier(AggregativeQuantifier, ABC):
+
+

+[docs]
+class AggregativeCrispQuantifier(AggregativeQuantifier, ABC):
     """
-    Abstract class for quantification methods that base their estimations on the aggregation of crips decisions
+    Abstract class for quantification methods that base their estimations on the aggregation of crisp decisions
     as returned by a hard classifier. Aggregative crisp quantifiers thus extend Aggregative
     Quantifiers by implementing specifications about crisp predictions.
     """
 
-    def _classifier_method(self):
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. For crisp quantifiers, the method
         is 'predict', that returns an array of shape `(n_instances,)` of label predictions.
@@ -337,7 +709,10 @@
         return 'predict'

 
 
-
[docs]class AggregativeSoftQuantifier(AggregativeQuantifier, ABC):
+
+

+[docs]
+class AggregativeSoftQuantifier(AggregativeQuantifier, ABC):
     """
     Abstract class for quantification methods that base their estimations on the aggregation of posterior
     probabilities as returned by a probabilistic classifier.
@@ -345,7 +720,7 @@
     about soft predictions.
     """
 
-    def _classifier_method(self):
+    def _classifier_method(self):
         """
         Name of the method that must be used for issuing label predictions. For probabilistic quantifiers, the method
         is 'predict_proba', that returns an array of shape `(n_instances, n_dimensions,)` with posterior
@@ -355,7 +730,7 @@
         """
         return 'predict_proba'
 
-    def _check_classifier(self, adapt_if_necessary=False):
+    def _check_classifier(self, adapt_if_necessary=False):
         """
         Guarantees that the underlying classifier implements the method indicated by the :meth:`_classifier_method`.
         In case it does not, the classifier is calibrated (by means of the Platt's calibration method implemented by
@@ -367,8 +742,8 @@
         """
         if not hasattr(self.classifier, self._classifier_method()):
             if adapt_if_necessary:
-                print(f'warning: The learner {self.classifier.__class__.__name__} does not seem to be '
-                      f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).')
+                warnings.warn(f'The learner {self.classifier.__class__.__name__} does not seem to be '
+                              f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).')
                 self.classifier = CalibratedClassifierCV(self.classifier, cv=5)
             else:
                 raise AssertionError(f'error: The learner {self.classifier.__class__.__name__} does not '
@@ -376,103 +751,247 @@
                                      f'fit_classifier is set to False')

 
 
-
[docs]class BinaryAggregativeQuantifier(AggregativeQuantifier, BinaryQuantifier):
-    
+
+

+[docs]
+class BinaryAggregativeQuantifier(AggregativeQuantifier, BinaryQuantifier):
+
     @property
-    def pos_label(self):
+    def pos_label(self):
         return self.classifier.classes_[1]
 
     @property
-    def neg_label(self):
+    def neg_label(self):
         return self.classifier.classes_[0]
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True, val_split=None):
-        self._check_binary(data, self.__class__.__name__)
-        return super().fit(data, fit_classifier, val_split)

+

+[docs]
+    def fit(self, X, y):
+        self._check_binary(y, self.__class__.__name__)
+        return super().fit(X, y)

+

+
 
 
 # Methods
 # ------------------------------------
-
[docs]class CC(AggregativeCrispQuantifier):
+

+[docs]
+class CC(AggregativeCrispQuantifier):
     """
-    The most basic Quantification method. One that simply classifies all instances and counts how many have been
-    attributed to each of the classes in order to compute class prevalence estimates.
+    `Classify & Count` (CC), the most basic quantification method, one that
+    simply classifies all instances and counts how many have been attributed to
+    each class in order to compute class prevalence estimates. This baseline is
+    the unadjusted estimator discussed, among others, in
+    `Forman, G. (2008). Quantifying counts and costs via classification.
+    Data Mining and Knowledge Discovery, 17, 164-206
+    <https://link.springer.com/article/10.1007/s10618-008-0097-y>`_.
+
+    :param classifier: a sklearn's Estimator that generates a classifier
+    """
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True):
+        super().__init__(classifier, fit_classifier, val_split=None)
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Nothing to do here!
+
+        :param classif_predictions: unused
+        :param labels: unused
+        """
+        pass

+
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
+        """
+        Computes class prevalence estimates by counting the prevalence of each of the predicted labels.
+
+        :param classif_predictions: array-like with classifier predictions
+        :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
+        """
+        return F.prevalence_from_labels(classif_predictions, self.classes_)

+

+
+
+
+

+[docs]
+class PCC(AggregativeSoftQuantifier):
+    """
+    `Probabilistic Classify & Count` (PCC), the probabilistic variant of CC
+    that relies on the posterior probabilities returned by a probabilistic
+    classifier, introduced in
+    `Bella, A., Ferri, C., Hernández-Orallo, J., and Ramírez-Quintana, M.J.
+    (2010). Quantification via probability estimators. In Proceedings of the
+    2010 IEEE International Conference on Data Mining (ICDM 2010)
+    <https://ieeexplore.ieee.org/abstract/document/5694031>`_.
 
     :param classifier: a sklearn's Estimator that generates a classifier
     """
 
-    def __init__(self, classifier: BaseEstimator):
-        self.classifier = classifier
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True, val_split=None):
+        super().__init__(classifier, fit_classifier, val_split=val_split)
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Nothing to do here!
 
-        :param classif_predictions: this is actually None
+        :param classif_predictions: unused
+        :param labels: unused
         """
         pass

 
-
[docs]    def aggregate(self, classif_predictions: np.ndarray):
-        """
-        Computes class prevalence estimates by counting the prevalence of each of the predicted labels.
 
-        :param classif_predictions: array-like with label predictions
-        :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
-        """
-        return F.prevalence_from_labels(classif_predictions, self.classes_)

+

+[docs]
+    def aggregate(self, classif_posteriors):
+        return F.prevalence_from_probabilities(classif_posteriors, binarize=False)

+

 
 
-
[docs]class ACC(AggregativeCrispQuantifier):
+
+

+[docs]
+class ACC(AggregativeCrispQuantifier):
     """
-    `Adjusted Classify & Count <https://link.springer.com/article/10.1007/s10618-008-0097-y>`_,
-    the "adjusted" variant of :class:`CC`, that corrects the predictions of CC
-    according to the `misclassification rates`.
+    `Adjusted Classify & Count` (ACC), the "adjusted" variant of :class:`CC`
+    that corrects the predictions of CC according to the
+    misclassification rates, originally proposed in
+    `Forman, G. (2008). Quantifying counts and costs via classification.
+    Data Mining and Knowledge Discovery, 17, 164-206
+    <https://link.springer.com/article/10.1007/s10618-008-0097-y>`_.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+    :param str method: adjustment method to be used:
+
+        * 'inversion': matrix inversion method based on the matrix equality :math:`P(C)=P(C|Y)P(Y)`,
+          which tries to invert :math:`P(C|Y)` matrix.
+        * 'invariant-ratio': invariant ratio estimator of `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation with the normalization condition.
+
+    :param str solver: indicates the method to use for solving the system of linear equations. Valid options are:
+
+        * 'exact-raise': tries to solve the system using matrix inversion. Raises an error if the matrix has rank
+          strictly less than `n_classes`.
+        * 'exact-cc': if the matrix is not of full rank, returns `p_c` as the estimates, which corresponds to
+          no adjustment (i.e., the classify and count method. See :class:`quapy.method.aggregative.CC`)
+        * 'exact': deprecated, defaults to 'exact-cc'
+        * 'minimize': minimizes the L2 norm of :math:`|Ax-B|`. This one generally works better, and is the
+          default parameter. More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of
+          Adjusted Classify and Count", on proceedings of the 2nd International Workshop on Learning to Quantify:
+          Methods and Applications (LQ 2022), ECML/PKDD 2022, Grenoble (France)
+          <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+
+    :param str norm: the method to use for normalization.
+
+        * `clip`, the values are clipped to the range [0,1] and then L1-normalized.
+        * `mapsimplex` projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `condsoftmax`, applies a softmax normalization only to prevalence vectors that lie outside the simplex
+
     :param n_jobs: number of parallel workers
-    :param solver: indicates the method to be used for obtaining the final estimates. The choice
-        'exact' comes down to solving the system of linear equations :math:`Ax=B` where `A` is a
-        matrix containing the class-conditional probabilities of the predictions (e.g., the tpr and fpr in 
-        binary) and `B` is the vector of prevalence values estimated via CC, as :math:`x=A^{-1}B`. This solution
-        might not exist for degenerated classifiers, in which case the method defaults to classify and count 
-        (i.e., does not attempt any adjustment).
-        Another option is to search for the prevalence vector that minimizes the L2 norm of :math:`|Ax-B|`. The latter
-        is achieved by indicating solver='minimize'. This one generally works better, and is the default parameter.
-        More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of Adjusted Classify and
-        Count", on proceedings of the 2nd International Workshop on Learning to Quantify: Methods and Applications
-        (LQ 2022), ECML/PKDD 2022, Grenoble (France) <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None, solver='minimize'):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier = True,
+            val_split = 5,
+            solver: Literal['minimize', 'exact', 'exact-raise', 'exact-cc'] = 'minimize',
+            method: Literal['inversion', 'invariant-ratio'] = 'inversion',
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+            n_jobs=None,
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
         self.solver = solver
+        self.method = method
+        self.norm = norm
 
-    def _check_init_parameters(self):
-        assert self.solver in ['exact', 'minimize'], "unknown solver; valid ones are 'exact', 'minimize'"
+    SOLVERS = ['exact', 'minimize', 'exact-raise', 'exact-cc']
+    METHODS = ['inversion', 'invariant-ratio']
+    NORMALIZATIONS = ['clip', 'mapsimplex', 'condsoftmax', None]
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    @classmethod
+    def newInvariantRatioEstimation(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, n_jobs=None):
+        """
+        Constructs a quantifier that implements the Invariant Ratio Estimator of
+        `Vaz et al. 2018 <https://jmlr.org/papers/v20/18-456.html>`_. This amounts
+        to setting method to 'invariant-ratio' and clipping to 'project'.
+
+        :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+            the one indicated in `qp.environ['DEFAULT_CLS']`
+
+        :param fit_classifier: whether to train the learner (default is True). Set to False if the
+            learner has been trained outside the quantifier.
+
+        :param val_split: specifies the data used for generating classifier predictions. This specification
+            can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+            be extracted from the training set; or as an integer (default 5), indicating that the predictions
+            are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+            for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+        :param n_jobs: number of parallel workers
+
+        :return: an instance of ACC configured so that it implements the Invariant Ratio Estimator
+        """
+        return ACC(classifier, fit_classifier=fit_classifier, val_split=val_split, method='invariant-ratio', norm='mapsimplex', n_jobs=n_jobs)

+
+
+    def _check_init_parameters(self):
+        if self.solver not in ACC.SOLVERS:
+            raise ValueError(f"unknown solver; valid ones are {ACC.SOLVERS}")
+        if self.method not in ACC.METHODS:
+            raise ValueError(f"unknown method; valid ones are {ACC.METHODS}")
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Estimates the misclassification rates.
-
-        :param classif_predictions: classifier predictions with true labels
+        :param classif_predictions: array-like with the predicted labels
+        :param labels: array-like with the true labels associated to each predicted label
         """
-        pred_labels, true_labels = classif_predictions.Xy
-        self.cc = CC(self.classifier)
-        self.Pte_cond_estim_ = self.getPteCondEstim(self.classifier.classes_, true_labels, pred_labels)

+        true_labels = labels
+        pred_labels = classif_predictions
+        self.cc = CC(self.classifier, fit_classifier=False)
+        self.Pte_cond_estim_ = ACC.getPteCondEstim(self.classifier.classes_, true_labels, pred_labels)

 
-
[docs]    @classmethod
-    def getPteCondEstim(cls, classes, y, y_):
-        # estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
-        # document that belongs to yj ends up being classified as belonging to yi
+
+

+[docs]
+    @classmethod
+    def getPteCondEstim(cls, classes, y, y_):
+        """
+        Estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
+        document that belongs to yj ends up being classified as belonging to yi
+
+        :param classes: array-like with the class names
+        :param y: array-like with the true labels
+        :param y_: array-like with the estimated labels
+        :return: np.ndarray
+        """
         conf = confusion_matrix(y, y_, labels=classes).T
         conf = conf.astype(float)
         class_counts = conf.sum(axis=0)
@@ -483,124 +1002,133 @@
                 conf[:, i] /= class_counts[i]
         return conf

 
-
[docs]    def aggregate(self, classif_predictions):
+
+

+[docs]
+    def aggregate(self, classif_predictions):
         prevs_estim = self.cc.aggregate(classif_predictions)
-        return ACC.solve_adjustment(self.Pte_cond_estim_, prevs_estim, solver=self.solver)

-
-
[docs]    @classmethod
-    def solve_adjustment(cls, PteCondEstim, prevs_estim, solver='exact'):
-        """
-        Solves the system linear system :math:`Ax = B` with :math:`A` = `PteCondEstim` and :math:`B` = `prevs_estim`
-
-        :param PteCondEstim: a `np.ndarray` of shape `(n_classes,n_classes,)` with entry `(i,j)` being the estimate
-            of :math:`P(y_i|y_j)`, that is, the probability that an instance that belongs to :math:`y_j` ends up being
-            classified as belonging to :math:`y_i`
-        :param prevs_estim: a `np.ndarray` of shape `(n_classes,)` with the class prevalence estimates
-        :param solver: indicates the method to use for solving the system of linear equations. Valid options are
-             'exact' (tries to solve the system --may fail if the misclassificatin matrix has rank < n_classes) or
-             'optim_minimize' (minimizes a norm --always exists). 
-        :return: an adjusted `np.ndarray` of shape `(n_classes,)` with the corrected class prevalence estimates
-        """
-
-        A = PteCondEstim
-        B = prevs_estim
-
-        if solver == 'exact':
-            # attempts an exact solution of the linear system (may fail)
-
-            try:
-                adjusted_prevs = np.linalg.solve(A, B)
-                adjusted_prevs = np.clip(adjusted_prevs, 0, 1)
-                adjusted_prevs /= adjusted_prevs.sum()
-            except np.linalg.LinAlgError:
-                adjusted_prevs = prevs_estim  # no way to adjust them!
-
-            return adjusted_prevs
-
-        elif solver == 'minimize':
-            # poses the problem as an optimization one, and tries to minimize the norm of the differences
-
-            def loss(prev):
-                return np.linalg.norm(A @ prev - B)
-
-            return F.optim_minimize(loss, n_classes=A.shape[0])

+        estimate = F.solve_adjustment(
+            class_conditional_rates=self.Pte_cond_estim_,
+            unadjusted_counts=prevs_estim,
+            solver=self.solver,
+            method=self.method,
+        )
+        return F.normalize_prevalence(estimate, method=self.norm)

+

 
 
-
[docs]class PCC(AggregativeSoftQuantifier):
+
+

+[docs]
+class PACC(AggregativeSoftQuantifier):
     """
-    `Probabilistic Classify & Count <https://ieeexplore.ieee.org/abstract/document/5694031>`_,
-    the probabilistic variant of CC that relies on the posterior probabilities returned by a probabilistic classifier.
+    `Probabilistic Adjusted Classify & Count` (PACC), the probabilistic
+    variant of ACC that relies on the posterior probabilities returned by a
+    probabilistic classifier, introduced in
+    `Bella, A., Ferri, C., Hernández-Orallo, J., and Ramírez-Quintana, M.J.
+    (2010). Quantification via probability estimators. In Proceedings of the
+    2010 IEEE International Conference on Data Mining (ICDM 2010)
+    <https://ieeexplore.ieee.org/abstract/document/5694031>`_.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    """
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
 
-    def __init__(self, classifier: BaseEstimator):
-        self.classifier = classifier
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        """
-        Nothing to do here!
-
-        :param classif_predictions: this is actually None
-        """
-        pass

-
-
[docs]    def aggregate(self, classif_posteriors):
-        return F.prevalence_from_probabilities(classif_posteriors, binarize=False)

-
-
-
[docs]class PACC(AggregativeSoftQuantifier):
-    """
-    `Probabilistic Adjusted Classify & Count <https://ieeexplore.ieee.org/abstract/document/5694031>`_,
-    the probabilistic variant of ACC that relies on the posterior probabilities returned by a probabilistic classifier.
-
-    :param classifier: a sklearn's Estimator that generates a classifier
     :param val_split: specifies the data used for generating classifier predictions. This specification
         can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
         be extracted from the training set; or as an integer (default 5), indicating that the predictions
         are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`). Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated.
-    :param n_jobs: number of parallel workers
-    :param solver: indicates the method to be used for obtaining the final estimates. The choice
-        'exact' comes down to solving the system of linear equations :math:`Ax=B` where `A` is a
-        matrix containing the class-conditional probabilities of the predictions (e.g., the tpr and fpr in
-        binary) and `B` is the vector of prevalence values estimated via CC, as :math:`x=A^{-1}B`. This solution
-        might not exist for degenerated classifiers, in which case the method defaults to classify and count
-        (i.e., does not attempt any adjustment).
-        Another option is to search for the prevalence vector that minimizes the L2 norm of :math:`|Ax-B|`. The latter
-        is achieved by indicating solver='minimize'. This one generally works better, and is the default parameter.
-        More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions of Adjusted Classify and
-        Count", on proceedings of the 2nd International Workshop on Learning to Quantify: Methods and Applications
-        (LQ 2022), ECML/PKDD 2022, Grenoble (France) <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
 
+    :param str method: adjustment method to be used:
+
+        * 'inversion': matrix inversion method based on the matrix equality :math:`P(C)=P(C|Y)P(Y)`,
+          which tries to invert `P(C|Y)` matrix.
+        * 'invariant-ratio': invariant ratio estimator of `Vaz et al. <https://jmlr.org/papers/v20/18-456.html>`_,
+          which replaces the last equation with the normalization condition.
+
+    :param str solver: the method to use for solving the system of linear equations. Valid options are:
+
+        * 'exact-raise': tries to solve the system using matrix inversion.
+          Raises an error if the matrix has rank strictly less than `n_classes`.
+        * 'exact-cc': if the matrix is not of full rank, returns `p_c` as the estimates, which
+          corresponds to no adjustment (i.e., the classify and count method. See :class:`quapy.method.aggregative.CC`)
+        * 'exact': deprecated, defaults to 'exact-cc'
+        * 'minimize': minimizes the L2 norm of :math:`|Ax-B|`. This one generally works better, and is the
+          default parameter. More details about this can be consulted in `Bunse, M. "On Multi-Class Extensions
+          of Adjusted Classify and Count", on proceedings of the 2nd International Workshop on Learning to
+          Quantify: Methods and Applications (LQ 2022), ECML/PKDD 2022, Grenoble (France)
+          <https://lq-2022.github.io/proceedings/CompleteVolume.pdf>`_.
+
+    :param str norm: the method to use for normalization.
+
+        * `clip`, the values are clipped to the range [0,1] and then L1-normalized.
+        * `mapsimplex` projects vectors onto the probability simplex. This implementation relies on
+          `Mathieu Blondel's projection_simplex_sort <https://gist.github.com/mblondel/6f3b7aaad90606b98f71>`_
+        * `condsoftmax`, applies a softmax normalization only to prevalence vectors that lie outside the simplex
+
+    :param n_jobs: number of parallel workers
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_jobs=None, solver='minimize'):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier=True,
+            val_split=5,
+            solver: Literal['minimize', 'exact', 'exact-raise', 'exact-cc'] = 'minimize',
+            method: Literal['inversion', 'invariant-ratio'] = 'inversion',
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+            n_jobs=None
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
         self.n_jobs = qp._get_njobs(n_jobs)
         self.solver = solver
+        self.method = method
+        self.norm = norm
 
-    def _check_init_parameters(self):
-        assert self.solver in ['exact', 'minimize'], "unknown solver; valid ones are 'exact', 'minimize'"
+    def _check_init_parameters(self):
+        if self.solver not in ACC.SOLVERS:
+            raise ValueError(f"unknown solver; valid ones are {ACC.SOLVERS}")
+        if self.method not in ACC.METHODS:
+            raise ValueError(f"unknown method; valid ones are {ACC.METHODS}")
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
         Estimates the misclassification rates
 
-        :param classif_predictions: classifier soft predictions with true labels
+        :param classif_predictions: array-like with posterior probabilities
+        :param labels: array-like with the true labels associated to each vector of posterior probabilities
         """
-        posteriors, true_labels = classif_predictions.Xy
-        self.pcc = PCC(self.classifier)
-        self.Pte_cond_estim_ = self.getPteCondEstim(self.classifier.classes_, true_labels, posteriors)

+        posteriors = classif_predictions
+        true_labels = labels
+        self.pcc = PCC(self.classifier, fit_classifier=False)
+        self.Pte_cond_estim_ = PACC.getPteCondEstim(self.classifier.classes_, true_labels, posteriors)

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         prevs_estim = self.pcc.aggregate(classif_posteriors)
-        return ACC.solve_adjustment(self.Pte_cond_estim_, prevs_estim, solver=self.solver)

 
-
[docs]    @classmethod
-    def getPteCondEstim(cls, classes, y, y_):
+        estimate = F.solve_adjustment(
+            class_conditional_rates=self.Pte_cond_estim_,
+            unadjusted_counts=prevs_estim,
+            solver=self.solver,
+            method=self.method,
+        )
+        return F.normalize_prevalence(estimate, method=self.norm)

+
+
+

+[docs]
+    @classmethod
+    def getPteCondEstim(cls, classes, y, y_):
         # estimate the matrix with entry (i,j) being the estimate of P(hat_yi|yj), that is, the probability that a
         # document that belongs to yj ends up being classified as belonging to yi
         n_classes = len(classes)
@@ -610,10 +1138,125 @@
             if idx.any():
                 confusion[i] = y_[idx].mean(axis=0)
 
-        return confusion.T

+        return confusion.T

+

 
 
-
[docs]class EMQ(AggregativeSoftQuantifier):
+
+

+[docs]
+class RLLS(AggregativeSoftQuantifier):
+    """
+    `Regularized Learning for Domain Adaptation under Label Shifts
+    <https://arxiv.org/abs/1903.09734>`_, used here as an aggregative
+    quantifier.
+
+    This implementation ports the regularized weight-estimation component of
+    RLLS to QuaPy's aggregative interface. It estimates label-shift ratios from
+    validation posteriors and source labels, then rescales the source
+    prevalence to obtain target prevalence estimates.
+
+    This method relies on the optional `cvxpy` dependency.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case
+        the classifier is taken to be the one indicated in
+        `qp.environ['DEFAULT_CLS']`
+    :param fit_classifier: whether to train the learner (default is True). Set
+        to False if the learner has been trained outside the quantifier.
+    :param val_split: specifies the data used for generating classifier
+        predictions. This specification can be made as float in (0, 1)
+        indicating the proportion of stratified held-out validation set to be
+        extracted from the training set; or as an integer (default 5),
+        indicating that the predictions are to be generated in a `k`-fold
+        cross-validation manner; or as a tuple `(X, y)` defining the specific
+        set of data to use for validation. This method requires source
+        predictions and therefore needs `val_split` whenever
+        `fit_classifier=True`.
+    :param mode: whether source- and target-domain quantities are estimated
+        from posterior probabilities (`soft`, default) or from argmax
+        predictions (`hard`)
+    :param alpha: multiplicative factor for the regularization level (default
+        0.01)
+    :param delta: confidence parameter used in the finite-sample regularizer
+        (default 0.05)
+    :param clip_weights: if True, clips negative importance weights to zero
+        before converting them into prevalence estimates
+    :param norm: the normalization method passed to
+        :func:`quapy.functional.normalize_prevalence`
+    """
+
+    def __init__(
+            self,
+            classifier: BaseEstimator = None,
+            fit_classifier=True,
+            val_split=5,
+            mode: Literal['soft', 'hard'] = 'soft',
+            alpha: float = 0.01,
+            delta: float = 0.05,
+            clip_weights: bool = True,
+            norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip',
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.mode = mode
+        self.alpha = alpha
+        self.delta = delta
+        self.clip_weights = clip_weights
+        self.norm = norm
+        self.last_w_ = None
+
+    def _check_init_parameters(self):
+        _get_cvxpy()
+        _rlls_check_mode(self.mode)
+        if not isinstance(self.alpha, (int, float)) or self.alpha < 0:
+            raise ValueError(f'expected a non-negative real value for alpha; found {self.alpha!r}')
+        if not isinstance(self.delta, (int, float)) or not (0 < self.delta < 1):
+            raise ValueError(f'expected delta to be in (0,1); found {self.delta!r}')
+        if self.norm not in ACC.NORMALIZATIONS:
+            raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}")
+        if self.fit_classifier and self.val_split is None:
+            raise ValueError(
+                'RLLS requires validation predictions for aggregation_fit; '
+                'please set val_split to an integer, float, or validation tuple'
+            )
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        if classif_predictions is None or labels is None:
+            raise ValueError('RLLS requires source posterior predictions and source labels')
+
+        self.train_prevalence_ = F.prevalence_from_labels(labels, classes=self.classes_)
+        self.C_zy_ = _rlls_joint_distribution(
+            classif_predictions,
+            labels,
+            self.classes_,
+            mode=self.mode,
+        )
+        self.pz_ = _rlls_predicted_marginal(classif_predictions, mode=self.mode)
+        self.rho_ = _rlls_compute_3deltaC(len(self.classes_), len(labels), self.delta)

+
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
+        qz = _rlls_predicted_marginal(classif_posteriors, mode=self.mode)
+        w = _rlls_compute_weights(
+            self.C_zy_,
+            qz,
+            self.pz_,
+            rho=self.alpha * self.rho_,
+            clip=self.clip_weights,
+        )
+        self.last_w_ = w
+        estimate = self.train_prevalence_ * w
+        return F.normalize_prevalence(estimate, method=self.norm)

+

+
+
+
+

+[docs]
+class EMQ(AggregativeSoftQuantifier):
     """
     `Expectation Maximization for Quantification <https://ieeexplore.ieee.org/abstract/document/6789744>`_ (EMQ),
     aka `Saerens-Latinne-Decaestecker` (SLD) algorithm.
@@ -626,108 +1269,206 @@
     prevalence, an estimate of it obtained via k-fold cross validation (instead of the true training prevalence),
     and to recalibrate the posterior probabilities of the classifier.
 
-    :param classifier: a sklearn's Estimator that generates a classifier
-    :param val_split: specifies the data used for generating classifier predictions. This specification
-        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
-        be extracted from the training set; or as an integer, indicating that the predictions
-        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
-        for `k`, default 5); or as a collection defining the specific set of data to use for validation.
-        Alternatively, this set can be specified at fit time by indicating the exact set of data
-        on which the predictions are to be generated. This hyperparameter is only meant to be used when the
-        heuristics are to be applied, i.e., if a recalibration is required. The default value is None (meaning
-        the recalibration is not required). In case this hyperparameter is set to a value other than None, but
-        the recalibration is not required (recalib=None), a warning message will be raised.
-    :param exact_train_prev: set to True (default) for using the true training prevalence as the initial observation;
-        set to False for computing the training prevalence as an estimate of it, i.e., as the expected
-        value of the posterior probabilities of the training instances.
-    :param recalib: a string indicating the method of recalibration.
-        Available choices include "nbvs" (No-Bias Vector Scaling), "bcts" (Bias-Corrected Temperature Scaling,
-        default), "ts" (Temperature Scaling), and "vs" (Vector Scaling). Default is None (no recalibration).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the classifier (default is True). Set to False if the
+        given classifier has already been trained.
+
+    :param val_split: specifies the data used for generating the classifier predictions on which the
+        aggregation function is to be trained. This specification can be made as float in (0, 1) indicating
+        the proportion of stratified held-out validation set to be extracted from the training set; or as
+        an integer (default 5), indicating that the predictions are to be generated in a `k`-fold
+        cross-validation manner (with this integer indicating the value for `k`); or as a tuple (X,y) defining
+        the specific set of data to use for validation. This hyperparameter is only meant to be used when
+        the heuristics are to be applied, i.e., if a calibration is required. The default value is None
+        (meaning the calibration is not required). In case this hyperparameter is set to a value other than
+        None, but the calibration is not required (calib=None), a warning message will be raised.
+
+    :param exact_train_prev: set to True (default) for using the true training prevalence as the initial
+        observation; set to False for computing the training prevalence as an estimate of it, i.e., as the
+        expected value of the posterior probabilities of the training instances.
+
+    :param calib: a string indicating the method of calibration.
+        Available choices include "nbvs" (No-Bias Vector Scaling), "bcts" (Bias-Corrected Temperature Scaling),
+        "ts" (Temperature Scaling), and "vs" (Vector Scaling). Default is None (no calibration).
+
+    :param on_calib_error: a string indicating the policy to follow in case the calibrator fails at runtime.
+        Options include "raise" (default), in which case a RuntimeException is raised; and "backup", in which
+        case the calibrator is silently skipped.
+
     :param n_jobs: number of parallel workers. Only used for recalibrating the classifier if `val_split` is set to
         an integer `k` --the number of folds.
     """
 
     MAX_ITER = 1000
     EPSILON = 1e-4
+    ON_CALIB_ERROR_VALUES = ['raise', 'backup']
+    CALIB_OPTIONS = [None, 'nbvs', 'bcts', 'ts', 'vs']
 
-    def __init__(self, classifier: BaseEstimator, val_split=None, exact_train_prev=True, recalib=None, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=None, exact_train_prev=True,
+                 calib=None, on_calib_error='raise', n_jobs=None):
+
+        assert calib in EMQ.CALIB_OPTIONS, \
+            f'invalid value for {calib=}; valid ones are {EMQ.CALIB_OPTIONS}'
+        assert on_calib_error in EMQ.ON_CALIB_ERROR_VALUES, \
+            f'invalid value for {on_calib_error=}; valid ones are {EMQ.ON_CALIB_ERROR_VALUES}'
+
+        super().__init__(classifier, fit_classifier, val_split)
         self.exact_train_prev = exact_train_prev
-        self.recalib = recalib
-        self.n_jobs = n_jobs
+        self.calib = calib
+        self.on_calib_error = on_calib_error
+        self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    @classmethod
-    def EMQ_BCTS(cls, classifier: BaseEstimator, n_jobs=None):
+

+[docs]
+    @classmethod
+    def EMQ_BCTS(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, on_calib_error="raise", n_jobs=None):
         """
         Constructs an instance of EMQ using the best configuration found in the `Alexandari et al. paper
         <http://proceedings.mlr.press/v119/alexandari20a.html>`_, i.e., one that relies on Bias-Corrected Temperature
-        Scaling (BCTS) as a recalibration function, and that uses an estimate of the training prevalence instead of
+        Scaling (BCTS) as a calibration function, and that uses an estimate of the training prevalence instead of
         the true training prevalence.
 
-        :param classifier: a sklearn's Estimator that generates a classifier
-        :param n_jobs: number of parallel workers.
+        :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+            the one indicated in `qp.environ['DEFAULT_CLS']`
+
+        :param fit_classifier: whether to train the learner (default is True). Set to False if the
+            learner has been trained outside the quantifier.
+
+        :param val_split: specifies the data used for generating classifier predictions. This specification
+            can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+            be extracted from the training set; or as an integer (default 5), indicating that the predictions
+            are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+            for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
+        :param on_calib_error: a string indicating the policy to follow in case the calibrator fails at runtime.
+            Options include "raise" (default), in which case a RuntimeException is raised; and "backup", in which
+            case the calibrator is silently skipped.
+
+        :param n_jobs: number of parallel workers. Only used for recalibrating the classifier if `val_split` is set to
+            an integer `k` --the number of folds.
+
         :return: An instance of EMQ with BCTS
         """
-        return EMQ(classifier, val_split=5, exact_train_prev=False, recalib='bcts', n_jobs=n_jobs)

+        return EMQ(classifier, fit_classifier=fit_classifier, val_split=val_split, exact_train_prev=False,
+                   calib='bcts', on_calib_error=on_calib_error, n_jobs=n_jobs)

 
-    def _check_init_parameters(self):
+
+    def _check_init_parameters(self):
         if self.val_split is not None:
-            if self.exact_train_prev and self.recalib is None:
-                raise RuntimeWarning(f'The parameter {self.val_split=} was specified for EMQ, while the parameters '
-                      f'{self.exact_train_prev=} and {self.recalib=}. This has no effect and causes an unnecessary '
-                      f'overload.')
+            if self.exact_train_prev and self.calib is None:
+                warnings.warn(f'The parameter {self.val_split=} was specified for EMQ, while the parameters '
+                              f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an '
+                              f'unnecessary overload.', RuntimeWarning)
         else:
-            if self.recalib is not None:
-                print(f'[warning] The parameter {self.recalib=} requires the val_split be different from None. '
-                      f'This parameter will be set to 5. To avoid this warning, set this value to a float value '
-                      f'indicating the proportion of training data to be used as validation, or to an integer '
-                      f'indicating the number of folds for kFCV.')
-                self.val_split=5
+            if self.calib is not None:
+                warnings.warn(f'The parameter {self.calib=} requires the val_split be different from None. '
+                              f'This parameter will be set to 5. To avoid this warning, set this value to a float value '
+                              f'indicating the proportion of training data to be used as validation, or to an integer '
+                              f'indicating the number of folds for kFCV.')
+                self.val_split = 5
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
-        Provides the posterior probabilities for the given instances. If the classifier was required
-        to be recalibrated, then these posteriors are recalibrated accordingly.
+        Provides the posterior probabilities for the given instances. The calibration function, if required,
+        has no effect in this step, and is only involved in the aggregate method.
 
-        :param instances: array-like of shape `(n_instances, n_dimensions,)`
+        :param X: array-like of shape `(n_instances, n_dimensions,)`
         :return: np.ndarray of shape `(n_instances, n_classes,)` with posterior probabilities
         """
-        posteriors = self.classifier.predict_proba(instances)
+        return self.classifier.predict_proba(X)

+
+
+

+[docs]
+    def classifier_fit_predict(self, X, y):
+        classif_predictions = super().classifier_fit_predict(X, y)
+        self.train_prevalence = F.prevalence_from_labels(y, classes=self.classes_)
+        return classif_predictions

+
+
+    def _fit_calibration(self, calibrator, P, y):
+        n_classes = len(self.classes_)
+
+        if not np.issubdtype(y.dtype, np.number):
+            y = np.searchsorted(self.classes_, y)
+
+        try:
+            self.calibration_function = calibrator(P, np.eye(n_classes)[y], posterior_supplied=True)
+        except Exception as e:
+            if self.on_calib_error == 'raise':
+                raise RuntimeError(f'calibration {self.calib} failed at fit time: {e}')
+            elif self.on_calib_error == 'backup':
+                self.calibration_function = lambda P: P
+
+    def _calibrate_if_requested(self, uncalib_posteriors):
         if hasattr(self, 'calibration_function') and self.calibration_function is not None:
-            posteriors = self.calibration_function(posteriors)
-        return posteriors

+            try:
+                calib_posteriors = self.calibration_function(uncalib_posteriors)
+            except Exception as e:
+                if self.on_calib_error == 'raise':
+                    raise RuntimeError(f'calibration {self.calib} failed at predict time: {e}')
+                elif self.on_calib_error == 'backup':
+                    calib_posteriors = uncalib_posteriors
+                else:
+                    raise ValueError(f'unexpected {self.on_calib_error=}; '
+                                     f'valid options are {EMQ.ON_CALIB_ERROR_VALUES}')
+            return calib_posteriors
+        return uncalib_posteriors
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        if self.recalib is not None:
-            P, y = classif_predictions.Xy
-            if self.recalib == 'nbvs':
-                calibrator = NoBiasVectorScaling()
-            elif self.recalib == 'bcts':
-                calibrator = TempScaling(bias_positions='all')
-            elif self.recalib == 'ts':
-                calibrator = TempScaling()
-            elif self.recalib == 'vs':
-                calibrator = VectorScaling()
-            else:
-                raise ValueError('invalid param argument for recalibration method; available ones are '
-                                 '"nbvs", "bcts", "ts", and "vs".')
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of EMQ. This comes down to recalibrating the posterior probabilities
+        ir requested.
 
-            self.calibration_function = calibrator(P, np.eye(data.n_classes)[y], posterior_supplied=True)
+        :param classif_predictions: array-like with the raw (i.e., uncalibrated) posterior probabilities
+            returned by the classifier
+        :param labels: array-like with the true labels associated to each classifier prediction
+        """
+        P = classif_predictions
+        y = labels
 
-        if self.exact_train_prev:
-            self.train_prevalence = data.prevalence()
-        else:
-            train_posteriors = classif_predictions.X
-            if self.recalib is not None:
-                train_posteriors = self.calibration_function(train_posteriors)
-            self.train_prevalence = F.prevalence_from_probabilities(train_posteriors)

+        requires_predictions = (self.calib is not None) or (not self.exact_train_prev)
+        if P is None and requires_predictions:
+            # classifier predictions were not generated because val_split=None
+            raise ValueError(self.__class__.__name__ +
+                             ": Classifier predictions for the aggregative fit were not generated because "
+                             "val_split=None. This usually happens when you enable calibrations or heuristics "
+                             "during model selection but left val_split set to its default value (None). "
+                             "Please provide one of the following values for val_split: (i) an integer >1 "
+                             "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. "
+                             "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit "
+                             "validation data")
 
-
[docs]    def aggregate(self, classif_posteriors, epsilon=EPSILON):
+        if self.calib is not None:
+            calibrator = _get_abstention_calibrators().get(self.calib, None)
+
+            if calibrator is None:
+                raise ValueError(f'invalid value for {self.calib=}; valid ones are {EMQ.CALIB_OPTIONS}')
+
+            self._fit_calibration(calibrator, P, y)
+
+        if not self.exact_train_prev:
+            P = self._calibrate_if_requested(P)
+            self.train_prevalence = F.prevalence_from_probabilities(P)

+
+
+

+[docs]
+    def aggregate(self, classif_posteriors, epsilon=EPSILON):
+        classif_posteriors = self._calibrate_if_requested(classif_posteriors)
         priors, posteriors = self.EM(self.train_prevalence, classif_posteriors, epsilon)
         return priors

 
-
[docs]    def predict_proba(self, instances, epsilon=EPSILON):
+
+

+[docs]
+    def predict_proba(self, instances, epsilon=EPSILON):
         """
         Returns the posterior probabilities updated by the EM algorithm.
 
@@ -736,11 +1477,15 @@
         :return: np.ndarray of shape `(n_instances, n_classes)`
         """
         classif_posteriors = self.classify(instances)
+        classif_posteriors = self._calibrate_if_requested(classif_posteriors)
         priors, posteriors = self.EM(self.train_prevalence, classif_posteriors, epsilon)
         return posteriors

 
-
[docs]    @classmethod
-    def EM(cls, tr_prev, posterior_probabilities, epsilon=EPSILON):
+
+

+[docs]
+    @classmethod
+    def EM(cls, tr_prev, posterior_probabilities, epsilon=EPSILON):
         """
         Computes the `Expectation Maximization` routine.
 
@@ -754,6 +1499,11 @@
         """
         Px = posterior_probabilities
         Ptr = np.copy(tr_prev)
+
+        if np.prod(Ptr) == 0:  # some entry is 0; we should smooth the values to avoid 0 division
+            Ptr += epsilon
+            Ptr /= Ptr.sum()
+
         qs = np.copy(Ptr)  # qs (the running estimate) is initialized as the training prevalence
 
         s, converged = 0, False
@@ -773,12 +1523,16 @@
             s += 1
 
         if not converged:
-            print('[warning] the method has reached the maximum number of iterations; it might have not converged')
+            warnings.warn('the method has reached the maximum number of iterations; it might have not converged')
 
-        return qs, ps

+        return qs, ps

+

 
 
-
[docs]class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `Hellinger Distance y <https://www.sciencedirect.com/science/article/pii/S0020025512004069>`_ (HDy).
     HDy is a probabilistic method for training binary quantifiers, that models quantification as the problem of
@@ -788,27 +1542,36 @@
     class-conditional distributions of the posterior probabilities returned for the positive and negative validation
     examples, respectively. The parameters of the mixture thus represent the estimates of the class prevalence values.
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    This dedicated class is kept for backward compatibility as the historical
+    HDy implementation. The same historical preset is also available as
+    :meth:`DMy.HDy`.
+
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
-        Trains a HDy quantifier.
+        Trains the aggregation function of HDy.
 
-        :param data: the training set
-        :param fit_classifier: set to False to bypass the training (the learner is assumed to be already fit)
-        :param val_split: either a float in (0,1) indicating the proportion of training instances to use for
-         validation (e.g., 0.3 for using 30% of the training set as validation data), or a
-         :class:`quapy.data.base.LabelledCollection` indicating the validation set itself
-        :return: self
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
         """
-        P, y = classif_predictions.Xy
+        P, y = classif_predictions, labels
         Px = P[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
@@ -816,16 +1579,17 @@
         # pre-compute the histogram for positive and negative examples
         self.bins = np.linspace(10, 110, 11, dtype=int)  # [10, 20, 30, ..., 100, 110]
 
-        def hist(P, bins):
+        def hist(P, bins):
             h = np.histogram(P, bins=bins, range=(0, 1), density=True)[0]
             return h / h.sum()
 
         self.Pxy1_density = {bins: hist(self.Pxy1, bins) for bins in self.bins}
-        self.Pxy0_density = {bins: hist(self.Pxy0, bins) for bins in self.bins}
+        self.Pxy0_density = {bins: hist(self.Pxy0, bins) for bins in self.bins}

 
-        return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+

+[docs]
+    def aggregate(self, classif_posteriors):
         # "In this work, the number of bins b used in HDx and HDy was chosen from 10 to 110 in steps of 10,
         # and the final estimated a priori probability was taken as the median of these 11 estimates."
         # (González-Castro, et al., 2013).
@@ -833,9 +1597,6 @@
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
 
         prev_estimations = []
-        # for bins in np.linspace(10, 110, 11, dtype=int):  #[10, 20, 30, ..., 100, 110]
-        # Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)
-        # Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)
         for bins in self.bins:
             Pxy0_density = self.Pxy0_density[bins]
             Pxy1_density = self.Pxy1_density[bins]
@@ -845,44 +1606,59 @@
             # the authors proposed to search for the prevalence yielding the best matching as a linear search
             # at small steps (modern implementations resort to an optimization procedure,
             # see class DistributionMatching)
-            prev_selected, min_dist = None, None
-            for prev in F.prevalence_linspace(n_prevalences=101, repeats=1, smooth_limits_epsilon=0.0):
-                Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density
-                hdy = F.HellingerDistance(Px_train, Px_test)
-                if prev_selected is None or hdy < min_dist:
-                    prev_selected, min_dist = prev, hdy
-            prev_estimations.append(prev_selected)
+            def loss(prev):
+                class1_prev = prev[1]
+                Px_train = class1_prev * Pxy1_density + (1 - class1_prev) * Pxy0_density
+                return F.HellingerDistance(Px_train, Px_test)
+
+            prev_estimations.append(F.linear_search(loss, n_classes=2)[1])
 
         class1_prev = np.median(prev_estimations)
-        return F.as_binary_prevalence(class1_prev)

+        return F.as_binary_prevalence(class1_prev)

+

 
 
-
[docs]class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `DyS framework <https://ojs.aaai.org/index.php/AAAI/article/view/4376>`_ (DyS).
     DyS is a generalization of HDy method, using a Ternary Search in order to find the prevalence that
     minimizes the distance between distributions.
     Details for the ternary search have been got from <https://dl.acm.org/doi/pdf/10.1145/3219819.3220059>
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     :param n_bins: an int with the number of bins to use to compute the histograms.
+
     :param divergence: a str indicating the name of divergence (currently supported ones are "HD" or "topsoe"), or a
         callable function computes the divergence between two distributions (two equally sized arrays).
+
     :param tol: a float with the tolerance for the ternary search algorithm.
+
     :param n_jobs: number of parallel workers.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5, n_bins=8, divergence: Union[str, Callable]= 'HD', tol=1e-05, n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_bins=8,
+                 divergence: Union[str, Callable] = 'HD', tol=1e-05, n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.tol = tol
         self.divergence = divergence
         self.n_bins = n_bins
-        self.n_jobs = n_jobs
+        self.n_jobs = qp._get_njobs(n_jobs)
 
-    def _ternary_search(self, f, left, right, tol):
+    def _ternary_search(self, f, left, right, tol):
         """
         Find maximum of unimodal function f() within [left, right]
         """
@@ -898,8 +1674,16 @@
         # Left and right are the current bounds; the maximum is between them
         return (left + right) / 2
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        Px, y = classif_predictions.Xy
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of DyS.
+
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
+        """
+        Px, y = classif_predictions, labels
         Px = Px[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
@@ -907,92 +1691,156 @@
         self.Pxy0_density = np.histogram(self.Pxy0, bins=self.n_bins, range=(0, 1), density=True)[0]
         return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
 
         Px_test = np.histogram(Px, bins=self.n_bins, range=(0, 1), density=True)[0]
         divergence = get_divergence(self.divergence)
 
-        def distribution_distance(prev):
+        def distribution_distance(prev):
             Px_train = prev * self.Pxy1_density + (1 - prev) * self.Pxy0_density
             return divergence(Px_train, Px_test)
-            
+
         class1_prev = self._ternary_search(f=distribution_distance, left=0, right=1, tol=self.tol)
-        return F.as_binary_prevalence(class1_prev)

+        return F.as_binary_prevalence(class1_prev)

+

 
 
-
[docs]class SMM(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
+
+

+[docs]
+class SMM(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
     """
     `SMM method <https://ieeexplore.ieee.org/document/9260028>`_ (SMM).
     SMM is a simplification of matching distribution methods where the representation of the examples
     is created using the mean instead of a histogram (conceptually equivalent to PACC).
 
-    :param classifier: a sklearn's Estimator that generates a binary classifier.
-    :param val_split: a float in range (0,1) indicating the proportion of data to be used as a stratified held-out
-        validation distribution, or a :class:`quapy.data.base.LabelledCollection` (the split itself), or an integer indicating the number of folds (default 5)..
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
     """
 
-    def __init__(self, classifier: BaseEstimator, val_split=5):
-        self.classifier = classifier
-        self.val_split = val_split
-      
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
-        Px, y = classif_predictions.Xy
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5):
+        super().__init__(classifier, fit_classifier, val_split)
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Trains the aggregation function of SMM.
+
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
+        """
+        Px, y = classif_predictions, labels
         Px = Px[:, self.pos_label]  # takes only the P(y=+1|x)
         self.Pxy1 = Px[y == self.pos_label]
         self.Pxy0 = Px[y == self.neg_label]
-        self.Pxy1_mean = np.mean(self.Pxy1)  # equiv. TPR 
+        self.Pxy1_mean = np.mean(self.Pxy1)  # equiv. TPR
         self.Pxy0_mean = np.mean(self.Pxy0)  # equiv. FPR
         return self

 
-
[docs]    def aggregate(self, classif_posteriors):
+
+

+[docs]
+    def aggregate(self, classif_posteriors):
         Px = classif_posteriors[:, self.pos_label]  # takes only the P(y=+1|x)
         Px_mean = np.mean(Px)
-     
-        class1_prev = (Px_mean - self.Pxy0_mean)/(self.Pxy1_mean - self.Pxy0_mean)
-        return F.as_binary_prevalence(class1_prev, clip_if_necessary=True)

+
+        class1_prev = (Px_mean - self.Pxy0_mean) / (self.Pxy1_mean - self.Pxy0_mean)
+        return F.as_binary_prevalence(class1_prev, clip_if_necessary=True)

+

 
 
-
[docs]class DMy(AggregativeSoftQuantifier):
+
+

+[docs]
+class DMy(AggregativeSoftQuantifier):
     """
     Generic Distribution Matching quantifier for binary or multiclass quantification based on the space of posterior
     probabilities. This implementation takes the number of bins, the divergence, and the possibility to work on CDF
     as hyperparameters.
 
-    :param classifier: a `sklearn`'s Estimator that generates a probabilistic classifier
-    :param val_split: indicates the proportion of data to be used as a stratified held-out validation set to model the
-        validation distribution.
-        This parameter can be indicated as a real value (between 0 and 1), representing a proportion of
-        validation data, or as an integer, indicating that the validation distribution should be estimated via
-        `k`-fold cross validation (this integer stands for the number of folds `k`, defaults 5), or as a
-        :class:`quapy.data.base.LabelledCollection` (the split itself).
+    :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be
+        the one indicated in `qp.environ['DEFAULT_CLS']`
+
+    :param fit_classifier: whether to train the learner (default is True). Set to False if the
+        learner has been trained outside the quantifier.
+
+    :param val_split: specifies the data used for generating classifier predictions. This specification
+        can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to
+        be extracted from the training set; or as an integer (default 5), indicating that the predictions
+        are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value
+        for `k`); or as a tuple (X,y) defining the specific set of data to use for validation.
+
     :param nbins: number of bins used to discretize the distributions (default 8)
+
     :param divergence: a string representing a divergence measure (currently, "HD" and "topsoe" are implemented)
         or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger
         Distance)
+
     :param cdf: whether to use CDF instead of PDF (default False)
+
+    :param search: string indicating the search strategy used to estimate the prevalence values.
+        Valid options are `optim_minimize` (default, works for binary and multiclass problems),
+        `linear_search` (binary only), and `ternary_search` (binary only)
+
     :param n_jobs: number of parallel workers (default None)
     """
 
-    def __init__(self, classifier, val_split=5, nbins=8, divergence: Union[str, Callable]='HD',
-                 cdf=False, search='optim_minimize', n_jobs=None):
-        self.classifier = classifier
-        self.val_split = val_split
+    def __init__(self, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, nbins=8,
+                 divergence: Union[str, Callable] = 'HD', cdf=False, search='optim_minimize', n_jobs=None):
+        super().__init__(classifier, fit_classifier, val_split)
         self.nbins = nbins
         self.divergence = divergence
         self.cdf = cdf
         self.search = search
-        self.n_jobs = n_jobs
+        self.n_jobs = qp._get_njobs(n_jobs)
 
-    # @classmethod
-    # def HDy(cls, classifier, val_split=5, n_jobs=None):
-    #     from quapy.method.meta import MedianEstimator
-    #
-    #     hdy = DMy(classifier=classifier, val_split=val_split, search='linear_search', divergence='HD')
-    #     hdy = AggregativeMedianEstimator(hdy, param_grid={'nbins': np.linspace(10, 110, 11).astype(int)}, n_jobs=n_jobs)
-    #     return hdy
+

+[docs]
+    @classmethod
+    def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None):
+        """
+        Historical HDy preset expressed as a configuration of :class:`DMy`.
 
-    def _get_distributions(self, posteriors):
+        This preset reproduces the original HDy setup by using Hellinger
+        distance, PDF matching, linear search, and a median sweep over
+        `nbins` in `[10, 20, ..., 110]`.
+
+        :param classifier: a scikit-learn's BaseEstimator, or None
+        :param fit_classifier: whether to train the learner
+        :param val_split: validation specification for generating posteriors
+        :param n_jobs: number of parallel workers
+        :return: an instance of :class:`AggregativeMedianEstimator` configured
+            to reproduce the historical HDy preset
+        """
+        base = cls(
+            classifier=classifier,
+            fit_classifier=fit_classifier,
+            val_split=val_split,
+            nbins=10,
+            divergence='HD',
+            cdf=False,
+            search='linear_search',
+            n_jobs=n_jobs,
+        )
+        param_grid = {'nbins': np.linspace(10, 110, 11, dtype=int)}
+        return AggregativeMedianEstimator(base_quantifier=base, param_grid=param_grid, n_jobs=n_jobs)

+
+
+    def _get_distributions(self, posteriors):
         histograms = []
         post_dims = posteriors.shape[1]
         if post_dims == 2:
@@ -1003,38 +1851,42 @@
             histograms.append(hist)
 
         counts = np.vstack(histograms)
-        distributions = counts/counts.sum(axis=1)[:,np.newaxis]
+        distributions = counts / counts.sum(axis=1)[:, np.newaxis]
         if self.cdf:
             distributions = np.cumsum(distributions, axis=1)
         return distributions
 
-
[docs]    def aggregation_fit(self, classif_predictions: LabelledCollection, data: LabelledCollection):
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
         """
-        Trains the classifier (if requested) and generates the validation distributions out of the training data.
+        Trains the aggregation function of a distribution matching method. This comes down to generating the
+        validation distributions out of the training data.
         The validation distributions have shape `(n, ch, nbins)`, with `n` the number of classes, `ch` the number of
         channels, and `nbins` the number of bins. In particular, let `V` be the validation distributions; then `di=V[i]`
         are the distributions obtained from training data labelled with class `i`; while `dij = di[j]` is the discrete
         distribution of posterior probabilities `P(Y=j|X=x)` for training data labelled with class `i`, and `dij[k]`
         is the fraction of instances with a value in the `k`-th bin.
 
-        :param data: the training set
-        :param fit_classifier: set to False to bypass the training (the learner is assumed to be already fit)
-        :param val_split: either a float in (0,1) indicating the proportion of training instances to use for
-         validation (e.g., 0.3 for using 30% of the training set as validation data), or a LabelledCollection
-         indicating the validation set itself, or an int indicating the number k of folds to be used in kFCV
-         to estimate the parameters
+        :param classif_predictions: array-like with the posterior probabilities returned by the classifier
+        :param labels: array-like with the true labels associated to each posterior
         """
-        posteriors, true_labels = classif_predictions.Xy
-        n_classes = len(self.classifier.classes_)
+        posteriors, true_labels = classif_predictions, labels
+        classes = self.classifier.classes_
+        n_classes = len(classes)
+        true_labels = _labels_to_indices(true_labels, classes)
 
         self.validation_distribution = qp.util.parallel(
             func=self._get_distributions,
-            args=[posteriors[true_labels==cat] for cat in range(n_classes)],
+            args=[posteriors[true_labels == cat] for cat in range(n_classes)],
             n_jobs=self.n_jobs,
             backend='threading'
         )

 
-
[docs]    def aggregate(self, posteriors: np.ndarray):
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
         """
         Searches for the mixture model parameter (the sought prevalence values) that yields a validation distribution
         (the mixture) that best matches the test distribution, in terms of the divergence measure of choice.
@@ -1048,17 +1900,21 @@
         test_distribution = self._get_distributions(posteriors)
         divergence = get_divergence(self.divergence)
         n_classes, n_channels, nbins = self.validation_distribution.shape
-        def loss(prev):
+
+        def loss(prev):
             prev = np.expand_dims(prev, axis=0)
-            mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes,-1)).reshape(n_channels, -1)
+            mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes, -1)).reshape(n_channels, -1)
             divs = [divergence(test_distribution[ch], mixture_distribution[ch]) for ch in range(n_channels)]
             return np.mean(divs)
 
-        return F.argmin_prevalence(loss, n_classes, method=self.search)

+        return F.argmin_prevalence(loss, n_classes, method=self.search)

+

 
 
 
-
[docs]def newELM(svmperf_base=None, loss='01', C=1):
+

+[docs]
+def newELM(svmperf_base=None, loss='01', C=1):
     """
     Explicit Loss Minimization (ELM) quantifiers.
     Quantifiers based on ELM represent a family of methods based on structured output learning;
@@ -1085,7 +1941,10 @@
     return CC(SVMperf(svmperf_base, loss=loss, C=C))

 
 
-
[docs]def newSVMQ(svmperf_base=None, C=1):
+
+

+[docs]
+def newSVMQ(svmperf_base=None, C=1):
     """
     SVM(Q) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the `Q` loss combining a
     classification-oriented loss and a quantification-oriented loss, as proposed by
@@ -1110,7 +1969,11 @@
     """
     return newELM(svmperf_base, loss='q', C=C)

 
-def newSVMKLD(svmperf_base=None, C=1):
+
+
+

+[docs]
+def newSVMKLD(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
     as proposed by `Esuli et al. 2015 <https://dl.acm.org/doi/abs/10.1145/2700406>`_.
@@ -1132,12 +1995,15 @@
     :return: returns an instance of CC set to work with SVMperf (with loss and C set properly) as the
         underlying classifier
     """
-    return newELM(svmperf_base, loss='kld', C=C)
+    return newELM(svmperf_base, loss='kld', C=C)

 
 
-
[docs]def newSVMKLD(svmperf_base=None, C=1):
+
+

+[docs]
+def newSVMNKLD(svmperf_base=None, C=1):
     """
-    SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
+    SVM(NKLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
     normalized via the logistic function, as proposed by
     `Esuli et al. 2015 <https://dl.acm.org/doi/abs/10.1145/2700406>`_.
     Equivalent to:
@@ -1160,7 +2026,11 @@
     """
     return newELM(svmperf_base, loss='nkld', C=C)

 
-
[docs]def newSVMAE(svmperf_base=None, C=1):
+
+
+

+[docs]
+def newSVMAE(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Absolute Error as first used by
     `Moreo and Sebastiani, 2021 <https://arxiv.org/abs/2011.02552>`_.
@@ -1184,7 +2054,11 @@
     """
     return newELM(svmperf_base, loss='mae', C=C)

 
-
[docs]def newSVMRAE(svmperf_base=None, C=1):
+
+
+

+[docs]
+def newSVMRAE(svmperf_base=None, C=1):
     """
     SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Relative Absolute Error as first
     used by `Moreo and Sebastiani, 2021 <https://arxiv.org/abs/2011.02552>`_.
@@ -1209,7 +2083,10 @@
     return newELM(svmperf_base, loss='mrae', C=C)

 
 
-
[docs]class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
+
+

+[docs]
+class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
     """
     Allows any binary quantifier to perform quantification on single-label datasets.
     The method maintains one binary quantifier for each class, and then l1-normalizes the outputs so that the
@@ -1218,56 +2095,78 @@
     `Gao and Sebastiani, 2016 <https://link.springer.com/content/pdf/10.1007/s13278-016-0327-z.pdf>`_.
 
     :param binary_quantifier: a quantifier (binary) that will be employed to work on multiclass model in a
-        one-vs-all manner
+        one-vs-all manner (default PACC(LogitsticRegression()))
     :param n_jobs: number of parallel workers
     :param parallel_backend: the parallel backend for joblib (default "loky"); this is helpful for some quantifiers
         (e.g., ELM-based ones) that cannot be run with multiprocessing, since the temp dir they create during fit will
         is removed and no longer available at predict time.
     """
 
-    def __init__(self, binary_quantifier, n_jobs=None, parallel_backend='multiprocessing'):
+    def __init__(self, binary_quantifier=None, n_jobs=None, parallel_backend='multiprocessing'):
+        if binary_quantifier is None:
+            binary_quantifier = PACC()
         assert isinstance(binary_quantifier, BaseQuantifier), \
-            f'{self.binary_quantifier} does not seem to be a Quantifier'
+            f'{binary_quantifier} does not seem to be a Quantifier'
         assert isinstance(binary_quantifier, AggregativeQuantifier), \
-            f'{self.binary_quantifier} does not seem to be of type Aggregative'
+            f'{binary_quantifier} does not seem to be of type Aggregative'
         self.binary_quantifier = binary_quantifier
         self.n_jobs = qp._get_njobs(n_jobs)
         self.parallel_backend = parallel_backend
 
-
[docs]    def classify(self, instances):
+

+[docs]
+    def classify(self, X):
         """
         If the base quantifier is not probabilistic, returns a matrix of shape `(n,m,)` with `n` the number of
         instances and `m` the number of classes. The entry `(i,j)` is a binary value indicating whether instance
-        `i `belongs to class `j`. The binary classifications are independent of each other, meaning that an instance
+        `i` belongs to class `j`. The binary classifications are independent of each other, meaning that an instance
         can end up be attributed to 0, 1, or more classes.
         If the base quantifier is probabilistic, returns a matrix of shape `(n,m,2)` with `n` the number of instances
         and `m` the number of classes. The entry `(i,j,1)` (resp. `(i,j,0)`) is a value in [0,1] indicating the
         posterior probability that instance `i` belongs (resp. does not belong) to class `j`. The posterior
         probabilities are independent of each other, meaning that, in general, they do not sum up to one.
 
-        :param instances: array-like
+        :param X: array-like
         :return: `np.ndarray`
         """
 
-        classif_predictions = self._parallel(self._delayed_binary_classification, instances)
+        classif_predictions = self._parallel(self._delayed_binary_classification, X)
         if isinstance(self.binary_quantifier, AggregativeSoftQuantifier):
             return np.swapaxes(classif_predictions, 0, 1)
         else:
             return classif_predictions.T

 
-
[docs]    def aggregate(self, classif_predictions):
+
+

+[docs]
+    def aggregate(self, classif_predictions):
         prevalences = self._parallel(self._delayed_binary_aggregate, classif_predictions)
         return F.normalize_prevalence(prevalences)

 
-    def _delayed_binary_classification(self, c, X):
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        self._parallel(self._delayed_binary_aggregate_fit, classif_predictions, labels)
+        return self

+
+
+    def _delayed_binary_classification(self, c, X):
         return self.dict_binary_quantifiers[c].classify(X)
 
-    def _delayed_binary_aggregate(self, c, classif_predictions):
+    def _delayed_binary_aggregate(self, c, classif_predictions):
         # the estimation for the positive class prevalence
-        return self.dict_binary_quantifiers[c].aggregate(classif_predictions[:, c])[1]

+        return self.dict_binary_quantifiers[c].aggregate(classif_predictions[:, c])[1]
+
+    def _delayed_binary_aggregate_fit(self, c, classif_predictions, labels):
+        # trains the aggregation function of the cth quantifier
+        return self.dict_binary_quantifiers[c].aggregation_fit(classif_predictions[:, c], labels == c)

 
 
-
[docs]class AggregativeMedianEstimator(BinaryQuantifier):
+
+

+[docs]
+class AggregativeMedianEstimator(BinaryQuantifier):
     """
     This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
     estimation returned by differently (hyper)parameterized base quantifiers.
@@ -1277,51 +2176,57 @@
     :param base_quantifier: the base, binary quantifier
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param param_grid: the grid or parameters towards which the median will be computed
-    :param n_jobs: number of parllel workes
+    :param n_jobs: number of parallel workers
     """
-    def __init__(self, base_quantifier: AggregativeQuantifier, param_grid: dict, random_state=None, n_jobs=None):
+
+    def __init__(self, base_quantifier: AggregativeQuantifier, param_grid: dict, random_state=None, n_jobs=None):
         self.base_quantifier = base_quantifier
         self.param_grid = param_grid
         self.random_state = random_state
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         return self.base_quantifier.get_params(deep)

 
-
[docs]    def set_params(self, **params):
+
+

+[docs]
+    def set_params(self, **params):
         self.base_quantifier.set_params(**params)

 
-    def _delayed_fit(self, args):
+
+    def _delayed_fit(self, args):
         with qp.util.temp_seed(self.random_state):
-            params, training = args
+            params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**params)
-            model.fit(training)
+            model.fit(X, y)
             return model
 
-    def _delayed_fit_classifier(self, args):
+    def _delayed_fit_classifier(self, args):
         with qp.util.temp_seed(self.random_state):
-            print('enter job')
-            cls_params, training, kwargs = args
+            cls_params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(training, **kwargs)
-            print('exit job')
-            return (model, predictions)
+            predictions, labels = model.classifier_fit_predict(X, y)
+            return (model, predictions, labels)
 
-    def _delayed_fit_aggregation(self, args):
+    def _delayed_fit_aggregation(self, args):
         with qp.util.temp_seed(self.random_state):
-            ((model, predictions), q_params), training = args
+            ((model, predictions, y), q_params) = args
             model = deepcopy(model)
             model.set_params(**q_params)
-            model.aggregation_fit(predictions, training)
+            model.aggregation_fit(predictions, y)
             return model
 
+

+[docs]
+    def fit(self, X, y):
+        import itertools
 
-
[docs]    def fit(self, training: LabelledCollection, **kwargs):
-        import itertools
-
-        self._check_binary(training, self.__class__.__name__)
+        self._check_binary(y, self.__class__.__name__)
 
         if isinstance(self.base_quantifier, AggregativeQuantifier):
             cls_configs, q_configs = qp.model_selection.group_params(self.param_grid)
@@ -1329,112 +2234,267 @@
             if len(cls_configs) > 1:
                 models_preds = qp.util.parallel(
                     self._delayed_fit_classifier,
-                    ((params, training, kwargs) for params in cls_configs),
+                    ((params, X, y) for params in cls_configs),
                     seed=qp.environ.get('_R_SEED', None),
                     n_jobs=self.n_jobs,
-                    asarray=False,
-                    backend='threading'
+                    asarray=False
                 )
             else:
-                print('only 1')
                 model = self.base_quantifier
                 model.set_params(**cls_configs[0])
-                predictions = model.classifier_fit_predict(training, **kwargs)
-                models_preds = [(model, predictions)]
+                predictions, labels = model.classifier_fit_predict(X, y)
+                models_preds = [(model, predictions, labels)]
 
             self.models = qp.util.parallel(
                 self._delayed_fit_aggregation,
-                ((setup, training) for setup in itertools.product(models_preds, q_configs)),
+                itertools.product(models_preds, q_configs),
                 seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                backend='threading'
+                n_jobs=self.n_jobs
             )
         else:
             configs = qp.model_selection.expand_grid(self.param_grid)
             self.models = qp.util.parallel(
                 self._delayed_fit,
-                ((params, training) for params in configs),
+                ((params, X, y) for params in configs),
                 seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                backend='threading'
+                n_jobs=self.n_jobs
             )
         return self

 
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
 
-
[docs]    def quantify(self, instances):
+    def _delayed_predict(self, args):
+        model, instances = args
+        return model.predict(instances)
+
+

+[docs]
+    def predict(self, instances):
         prev_preds = qp.util.parallel(
             self._delayed_predict,
             ((model, instances) for model in self.models),
             seed=qp.environ.get('_R_SEED', None),
-            n_jobs=self.n_jobs,
-            backend='threading'
+            n_jobs=self.n_jobs
         )
-        return np.median(prev_preds, axis=0)

+        return np.median(prev_preds, axis=0)

+

 
 
-#---------------------------------------------------------------
+
+

+[docs]
+class EDy(_EnergyDistanceCore, AggregativeSoftQuantifier):
+    """
+    Energy Distance y (EDy), a posterior-space distribution-matching quantifier
+    based on energy distance.
+
+    The method represents each class by the posterior-probability vectors
+    produced by a probabilistic classifier on validation data, and estimates the
+    test prevalence vector by matching the test posterior distribution against
+    the class-conditional validation distributions through an energy-distance
+    objective solved as a quadratic program. The method is therefore another
+    instance of the general mixture-matching view of quantification, but it
+    operates directly on posterior vectors rather than on histogram summaries.
+
+    This implementation works for binary and multiclass single-label
+    quantification and relies on the optional ``quadprog`` dependency. It was
+    adapted to QuaPy's current aggregative API from the original implementation
+    available in `quantificationlib <https://github.com/AICGijon/quantificationlib>`_,
+    and now shares its numerical core with the classifier-free
+    :class:`quapy.method.non_aggregative.EDx` variant.
+
+    The current implementation follows the energy-distance formulation discussed
+    in:
+
+    * Alberto Castaño, Laura Morán-Fernández, Jaime Alonso,
+      Verónica Bolón-Canedo, Amparo Alonso-Betanzos, and Juan José del Coz.
+      *An analysis of quantification methods based on matching distributions*.
+    * Hideko Kawakubo, Marthinus Christoffel du Plessis, and Masashi Sugiyama
+      (2016). *Computationally efficient class-prior estimation under class
+      balance change using energy distance*. IEICE Transactions on Information
+      and Systems, 99(1):176-186.
+
+    :param classifier: a scikit-learn ``BaseEstimator``, or ``None`` to use
+        ``qp.environ['DEFAULT_CLS']``
+    :param fit_classifier: whether to train the learner (default ``True``).
+        Set to ``False`` if the learner has already been trained outside the
+        quantifier
+    :param val_split: specification of the data used for generating validation
+        posterior probabilities. This can be an integer (default ``5``) for
+        k-fold cross-validation, a float in ``(0, 1)`` for a held-out split,
+        or a tuple ``(X, y)`` with explicit validation data
+    :param distance: distance used to compare posterior vectors. Valid string
+        aliases are ``'manhattan'`` (default) and ``'euclidean'``; a custom
+        callable compatible with pairwise-distance signatures can also be used
+    :param n_jobs: number of parallel workers (default ``None``, meaning the
+        value is taken from the environment)
+    """
+
+    def __init__(
+        self,
+        classifier: BaseEstimator = None,
+        fit_classifier: bool = True,
+        val_split=5,
+        distance: Union[str, Callable] = 'manhattan',
+        n_jobs=None,
+    ):
+        super().__init__(classifier, fit_classifier, val_split)
+        self.distance = distance
+        self.n_jobs = qp._get_njobs(n_jobs)
+        self.train_n_cls_i_ = None
+        self.train_distrib_ = None
+        self.K_ = None
+        self.G_ = None
+        self.C_ = None
+        self.b_ = None
+        self.a_ = None
+
+    def _check_init_parameters(self):
+        self._check_ed_init_parameters()
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        """
+        Estimate the class-conditional posterior distributions on validation
+        data and pre-compute the quadratic-program parameters that depend only
+        on the training side.
+
+        In EDy, the validation posteriors are not discretized into histograms.
+        Instead, each class is represented by the cloud of posterior vectors
+        observed for that class, and these clouds are then compared through the
+        selected pairwise distance.
+
+        :param classif_predictions: posterior probabilities returned by the
+            classifier on validation data
+        :param labels: true labels associated to each posterior vector
+        """
+        posteriors = np.asarray(classif_predictions, dtype=float)
+        labels = np.asarray(labels)
+        train_distrib = [posteriors[labels == class_] for class_ in self.classes_]
+        return self._fit_energy_model(train_distrib)

+
+
+

+[docs]
+    def aggregate(self, posteriors: np.ndarray):
+        """Estimate the prevalence vector for a test sample.
+
+        :param posteriors: posterior probabilities returned by the classifier
+            for the instances in the test sample
+        :return: a prevalence vector of shape ``(n_classes,)``
+        """
+        posteriors = np.asarray(posteriors, dtype=float)
+        return self._predict_energy(posteriors)

+

+
+
+
+# ---------------------------------------------------------------
 # imports
-#---------------------------------------------------------------
+# ---------------------------------------------------------------
 
-from . import _threshold_optim
+from . import _threshold_optim
 
 T50 = _threshold_optim.T50
 MAX = _threshold_optim.MAX
-X   = _threshold_optim.X
-MS  = _threshold_optim.MS
+X = _threshold_optim.X
+MS = _threshold_optim.MS
 MS2 = _threshold_optim.MS2
 
-
-from . import _kdey
+from . import _kdey
 
 KDEyML = _kdey.KDEyML
 KDEyHD = _kdey.KDEyHD
 KDEyCS = _kdey.KDEyCS
 
-#---------------------------------------------------------------
+
+# ---------------------------------------------------------------
 # aliases
-#---------------------------------------------------------------
+# ---------------------------------------------------------------
 
 ClassifyAndCount = CC
 AdjustedClassifyAndCount = ACC
 ProbabilisticClassifyAndCount = PCC
 ProbabilisticAdjustedClassifyAndCount = PACC
 ExpectationMaximizationQuantifier = EMQ
-DistributionMatchingY = DMy
 SLD = EMQ
+DistributionMatchingY = DMy
+EnergyDistanceY = EDy
 HellingerDistanceY = HDy
+HistoricalHDy = DMy.HDy
 MedianSweep = MS
 MedianSweep2 = MS2
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/base.html b/docs/build/html/_modules/quapy/method/base.html index 6288bd1..7142f34 100644 --- a/docs/build/html/_modules/quapy/method/base.html +++ b/docs/build/html/_modules/quapy/method/base.html @@ -1,133 +1,464 @@ + - - - - - quapy.method.base — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method.base — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.method.base

-from abc import ABCMeta, abstractmethod
-from copy import deepcopy
+import warnings
+from abc import ABCMeta, abstractmethod
+from copy import deepcopy
 
-from joblib import Parallel, delayed
-from sklearn.base import BaseEstimator
+from joblib import Parallel, delayed
+from sklearn.base import BaseEstimator
 
-import quapy as qp
-from quapy.data import LabelledCollection
-import numpy as np
+import quapy as qp
+from quapy.data import LabelledCollection
+import numpy as np
 
 
 # Base Quantifier abstract class
 # ------------------------------------
-
[docs]class BaseQuantifier(BaseEstimator):
+

+[docs]
+class BaseQuantifier(BaseEstimator):
     """
     Abstract Quantifier. A quantifier is defined as an object of a class that implements the method :meth:`fit` on
-    :class:`quapy.data.base.LabelledCollection`, the method :meth:`quantify`, and the :meth:`set_params` and
+    a pair X, y, the method :meth:`predict`, and the :meth:`set_params` and
     :meth:`get_params` for model selection (see :meth:`quapy.model_selection.GridSearchQ`)
     """
 
-
[docs]    @abstractmethod
-    def fit(self, data: LabelledCollection):
+

+[docs]
+    @abstractmethod
+    def fit(self, X, y):
         """
-        Trains a quantifier.
+        Generates a quantifier.
 
-        :param data: a :class:`quapy.data.base.LabelledCollection` consisting of the training data
+        :param X: array-like, the training instances
+        :param y: array-like, the labels
         :return: self
         """
         ...

 
-
[docs]    @abstractmethod
-    def quantify(self, instances):
+
+

+[docs]
+    @abstractmethod
+    def predict(self, X):
         """
         Generate class prevalence estimates for the sample's instances
 
-        :param instances: array-like
+        :param X: array-like, the test instances
         :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
         """
-        ...

+        ...

 
 
-
[docs]class BinaryQuantifier(BaseQuantifier):
+

+[docs]
+    def quantify(self, X):
+        """
+        Alias to :meth:`predict`, for old compatibility
+
+        :param X: array-like
+        :return: `np.ndarray` of shape `(n_classes,)` with class prevalence estimates.
+        """
+        return self.predict(X)

+

+
+
+
+

+[docs]
+class BinaryQuantifier(BaseQuantifier):
     """
     Abstract class of binary quantifiers, i.e., quantifiers estimating class prevalence values for only two classes
     (typically, to be interpreted as one class and its complement).
     """
 
-    def _check_binary(self, data: LabelledCollection, quantifier_name):
-        assert data.binary, f'{quantifier_name} works only on problems of binary classification. ' \
+    def _check_binary(self, y, quantifier_name):
+        n_classes = len(set(y))
+        assert n_classes==2, f'{quantifier_name} works only on problems of binary classification. ' \
                             f'Use the class OneVsAll to enable {quantifier_name} work on single-label data.'

 
 
-
[docs]class OneVsAll:
+
+

+[docs]
+class OneVsAll:
     pass

 
 
-
[docs]def newOneVsAll(binary_quantifier, n_jobs=None):
+
+

+[docs]
+def newOneVsAll(binary_quantifier: BaseQuantifier, n_jobs=None):
     assert isinstance(binary_quantifier, BaseQuantifier), \
         f'{binary_quantifier} does not seem to be a Quantifier'
     if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier):
@@ -136,77 +467,131 @@
         return OneVsAllGeneric(binary_quantifier, n_jobs)

 
 
-
[docs]class OneVsAllGeneric(OneVsAll, BaseQuantifier):
+
+

+[docs]
+class OneVsAllGeneric(OneVsAll, BaseQuantifier):
     """
     Allows any binary quantifier to perform quantification on single-label datasets. The method maintains one binary
-    quantifier for each class, and then l1-normalizes the outputs so that the class prevelence values sum up to 1.
+    quantifier for each class, and then l1-normalizes the outputs so that the class prevalence values sum up to 1.
     """
 
-    def __init__(self, binary_quantifier, n_jobs=None):
+    def __init__(self, binary_quantifier: BaseQuantifier, n_jobs=None):
         assert isinstance(binary_quantifier, BaseQuantifier), \
             f'{binary_quantifier} does not seem to be a Quantifier'
         if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier):
-            print('[warning] the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; '
-                  f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}')
+            warnings.warn('the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; '
+                           f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}')
         self.binary_quantifier = binary_quantifier
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def fit(self, data: LabelledCollection, fit_classifier=True):
-        assert not data.binary, f'{self.__class__.__name__} expect non-binary data'
-        assert fit_classifier == True, 'fit_classifier must be True'
+

+[docs]
+    def fit(self, X, y):
+        self.classes = sorted(np.unique(y))
+        assert len(self.classes)!=2, f'{self.__class__.__name__} expect non-binary data'
 
-        self.dict_binary_quantifiers = {c: deepcopy(self.binary_quantifier) for c in data.classes_}
-        self._parallel(self._delayed_binary_fit, data)
+        self.dict_binary_quantifiers = {c: deepcopy(self.binary_quantifier) for c in self.classes}
+        self._parallel(self._delayed_binary_fit, X, y)
         return self

 
-    def _parallel(self, func, *args, **kwargs):
+
+    def _parallel(self, func, *args, **kwargs):
         return np.asarray(
             Parallel(n_jobs=self.n_jobs, backend='threading')(
-                delayed(func)(c, *args, **kwargs) for c in self.classes_
+                delayed(func)(c, *args, **kwargs) for c in self.classes
             )
         )
 
-
[docs]    def quantify(self, instances):
-        prevalences = self._parallel(self._delayed_binary_predict, instances)
+

+[docs]
+    def predict(self, X):
+        prevalences = self._parallel(self._delayed_binary_predict, X)
         return qp.functional.normalize_prevalence(prevalences)

 
-    @property
-    def classes_(self):
-        return sorted(self.dict_binary_quantifiers.keys())
 
-    def _delayed_binary_predict(self, c, X):
-        return self.dict_binary_quantifiers[c].quantify(X)[1]
+    # @property
+    # def classes_(self):
+    #     return sorted(self.dict_binary_quantifiers.keys())
+
+    def _delayed_binary_predict(self, c, X):
+        return self.dict_binary_quantifiers[c].predict(X)[1]
+
+    def _delayed_binary_fit(self, c, X, y):
+        bindata = LabelledCollection(X, y == c, classes=[False, True])
+        self.dict_binary_quantifiers[c].fit(*bindata.Xy)

 
-    def _delayed_binary_fit(self, c, data):
-        bindata = LabelledCollection(data.instances, data.labels == c, classes=[False, True])
-        self.dict_binary_quantifiers[c].fit(bindata)
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/meta.html b/docs/build/html/_modules/quapy/method/meta.html index ca38440..fff036a 100644 --- a/docs/build/html/_modules/quapy/method/meta.html +++ b/docs/build/html/_modules/quapy/method/meta.html @@ -1,94 +1,394 @@ + - - - - - quapy.method.meta — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method.meta — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ +

Source code for quapy.method.meta

+import itertools
+import logging
+from copy import deepcopy
+from typing import Union, List
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import f1_score, make_scorer, accuracy_score
+from sklearn.model_selection import GridSearchCV, cross_val_predict
+from tqdm import tqdm
+
+import quapy as qp
+from quapy import functional as F
+from quapy.data import LabelledCollection
+from quapy.model_selection import GridSearchQ
+from quapy.method.base import BaseQuantifier, BinaryQuantifier
+from quapy.method.aggregative import CC, ACC, PACC, HDy, EMQ, AggregativeQuantifier, AggregativeSoftQuantifier
 
 try:
-    from . import _neural
+    from . import _neural
 except ModuleNotFoundError:
     _neural = None
 
@@ -99,7 +399,9 @@
     QuaNet = "QuaNet is not available due to missing torch package"
 
 
-
[docs]class MedianEstimator2(BinaryQuantifier):
+

+[docs]
+class MedianEstimator(BinaryQuantifier):
     """
     This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
     estimation returned by differently (hyper)parameterized base quantifiers.
@@ -109,159 +411,73 @@
     :param base_quantifier: the base, binary quantifier
     :param random_state: a seed to be set before fitting any base quantifier (default None)
     :param param_grid: the grid or parameters towards which the median will be computed
-    :param n_jobs: number of parllel workes
+    :param n_jobs: number of parallel workers
     """
-    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
+    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
         self.base_quantifier = base_quantifier
         self.param_grid = param_grid
         self.random_state = random_state
         self.n_jobs = qp._get_njobs(n_jobs)
 
-
[docs]    def get_params(self, deep=True):
+

+[docs]
+    def get_params(self, deep=True):
         return self.base_quantifier.get_params(deep)

 
-
[docs]    def set_params(self, **params):
+
+

+[docs]
+    def set_params(self, **params):
         self.base_quantifier.set_params(**params)

 
-    def _delayed_fit(self, args):
+
+    def _delayed_fit(self, args):
         with qp.util.temp_seed(self.random_state):
-            params, training = args
+            params, X, y = args
             model = deepcopy(self.base_quantifier)
             model.set_params(**params)
-            model.fit(training)
+            model.fit(X, y)
             return model
 
-
[docs]    def fit(self, training: LabelledCollection):
-        self._check_binary(training, self.__class__.__name__)
+

+[docs]
+    def fit(self, X, y):
+        self._check_binary(y, self.__class__.__name__)
 
         configs = qp.model_selection.expand_grid(self.param_grid)
         self.models = qp.util.parallel(
             self._delayed_fit,
-            ((params, training) for params in configs),
+            ((params, X, y) for params in configs),
             seed=qp.environ.get('_R_SEED', None),
-            n_jobs=self.n_jobs
+            n_jobs=self.n_jobs,
+            asarray=False
         )
         return self

 
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
 
-
[docs]    def quantify(self, instances):
+    def _delayed_predict(self, args):
+        model, instances = args
+        return model.predict(instances)
+
+

+[docs]
+    def predict(self, X):
         prev_preds = qp.util.parallel(
             self._delayed_predict,
-            ((model, instances) for model in self.models),
-            seed=qp.environ.get('_R_SEED', None),
-            n_jobs=self.n_jobs
-        )
-        prev_preds = np.asarray(prev_preds)
-        return np.median(prev_preds, axis=0)

-
-
-
[docs]class MedianEstimator(BinaryQuantifier):
-    """
-    This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
-    estimation returned by differently (hyper)parameterized base quantifiers.
-    The median of unit-vectors is only guaranteed to be a unit-vector for n=2 dimensions,
-    i.e., in cases of binary quantification.
-
-    :param base_quantifier: the base, binary quantifier
-    :param random_state: a seed to be set before fitting any base quantifier (default None)
-    :param param_grid: the grid or parameters towards which the median will be computed
-    :param n_jobs: number of parllel workes
-    """
-    def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
-        self.base_quantifier = base_quantifier
-        self.param_grid = param_grid
-        self.random_state = random_state
-        self.n_jobs = qp._get_njobs(n_jobs)
-
-
[docs]    def get_params(self, deep=True):
-        return self.base_quantifier.get_params(deep)

-
-
[docs]    def set_params(self, **params):
-        self.base_quantifier.set_params(**params)

-
-    def _delayed_fit(self, args):
-        with qp.util.temp_seed(self.random_state):
-            params, training = args
-            model = deepcopy(self.base_quantifier)
-            model.set_params(**params)
-            model.fit(training)
-            return model
-
-    def _delayed_fit_classifier(self, args):
-        with qp.util.temp_seed(self.random_state):
-            cls_params, training = args
-            model = deepcopy(self.base_quantifier)
-            model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(training, predict_on=model.val_split)
-            return (model, predictions)
-
-    def _delayed_fit_aggregation(self, args):
-        with qp.util.temp_seed(self.random_state):
-            ((model, predictions), q_params), training = args
-            model = deepcopy(model)
-            model.set_params(**q_params)
-            model.aggregation_fit(predictions, training)
-            return model
-
-
-
[docs]    def fit(self, training: LabelledCollection):
-        self._check_binary(training, self.__class__.__name__)
-
-        if isinstance(self.base_quantifier, AggregativeQuantifier):
-            cls_configs, q_configs = qp.model_selection.group_params(self.param_grid)
-
-            if len(cls_configs) > 1:
-                models_preds = qp.util.parallel(
-                    self._delayed_fit_classifier,
-                    ((params, training) for params in cls_configs),
-                    seed=qp.environ.get('_R_SEED', None),
-                    n_jobs=self.n_jobs,
-                    asarray=False
-                )
-            else:
-                model = self.base_quantifier
-                model.set_params(**cls_configs[0])
-                predictions = model.classifier_fit_predict(training, predict_on=model.val_split)
-                models_preds = [(model, predictions)]
-
-            self.models = qp.util.parallel(
-                self._delayed_fit_aggregation,
-                ((setup, training) for setup in itertools.product(models_preds, q_configs)),
-                seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                asarray=False
-            )
-        else:
-            configs = qp.model_selection.expand_grid(self.param_grid)
-            self.models = qp.util.parallel(
-                self._delayed_fit,
-                ((params, training) for params in configs),
-                seed=qp.environ.get('_R_SEED', None),
-                n_jobs=self.n_jobs,
-                asarray=False
-            )
-        return self

-
-    def _delayed_predict(self, args):
-        model, instances = args
-        return model.quantify(instances)
-
-
[docs]    def quantify(self, instances):
-        prev_preds = qp.util.parallel(
-            self._delayed_predict,
-            ((model, instances) for model in self.models),
+            ((model, X) for model in self.models),
             seed=qp.environ.get('_R_SEED', None),
             n_jobs=self.n_jobs,
             asarray=False
         )
         prev_preds = np.asarray(prev_preds)
-        return np.median(prev_preds, axis=0)

+        return np.median(prev_preds, axis=0)

+

 
 
-
[docs]class Ensemble(BaseQuantifier):
+
+

+[docs]
+class Ensemble(BaseQuantifier):
     VALID_POLICIES = {'ave', 'ptr', 'ds'} | qp.error.QUANTIFICATION_ERROR_NAMES
 
     """
@@ -301,7 +517,7 @@
     :param verbose: set to True (default is False) to get some information in standard output
     """
 
-    def __init__(self,
+    def __init__(self,
                  quantifier: BaseQuantifier,
                  size=50,
                  red_size=25,
@@ -326,17 +542,20 @@
         self.verbose = verbose
         self.max_sample_size = max_sample_size
 
-    def _sout(self, msg):
+    def _sout(self, msg):
         if self.verbose:
-            print('[Ensemble]' + msg)
+            logging.getLogger(__name__).info('[Ensemble] ' + msg)
 
-
[docs]    def fit(self, data: qp.data.LabelledCollection, val_split: Union[qp.data.LabelledCollection, float] = None):
+

+[docs]
+    def fit(self, X, y):
+
+        data = LabelledCollection(X, y)
 
         if self.policy == 'ds' and not data.binary:
             raise ValueError(f'ds policy is only defined for binary quantification, but this dataset is not binary')
 
-        if val_split is None:
-            val_split = self.val_split
+        val_split = self.val_split
 
         # randomly chooses the prevalences for each member of the ensemble (preventing classes with less than
         # min_pos positive examples)
@@ -367,20 +586,26 @@
         self._sout('Fit [Done]')
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         predictions = np.asarray(
-            qp.util.parallel(_delayed_quantify, ((Qi, instances) for Qi in self.ensemble), n_jobs=self.n_jobs)
+            qp.util.parallel(_delayed_quantify, ((Qi, X) for Qi in self.ensemble), n_jobs=self.n_jobs)
         )
 
         if self.policy == 'ptr':
             predictions = self._ptr_policy(predictions)
         elif self.policy == 'ds':
-            predictions = self._ds_policy(predictions, instances)
+            predictions = self._ds_policy(predictions, X)
 
         predictions = np.mean(predictions, axis=0)
         return F.normalize_prevalence(predictions)

 
-
[docs]    def set_params(self, **parameters):
+
+

+[docs]
+    def set_params(self, **parameters):
         """
         This function should not be used within :class:`quapy.model_selection.GridSearchQ` (is here for compatibility
         with the abstract class).
@@ -396,7 +621,10 @@
                                   f'or Ensemble(Q(GridSearchCV(l))) with Q a quantifier class that has a classifier '
                                   f'l optimized for classification (not recommended).')

 
-
[docs]    def get_params(self, deep=True):
+
+

+[docs]
+    def get_params(self, deep=True):
         """
         This function should not be used within :class:`quapy.model_selection.GridSearchQ` (is here for compatibility
         with the abstract class).
@@ -410,13 +638,14 @@
 
         raise NotImplementedError()

 
-    def _accuracy_policy(self, error_name):
+
+    def _accuracy_policy(self, error_name):
         """
         Selects the red_size best performant quantifiers in a static way (i.e., dropping all non-selected instances).
         For each model in the ensemble, the performance is measured in terms of _error_name_ on the quantification of
         the samples used for training the rest of the models in the ensemble.
         """
-        from quapy.evaluation import evaluate_on_samples
+        from quapy.evaluation import evaluate_on_samples
         error = qp.error.from_name(error_name)
         tests = [m[3] for m in self.ensemble]
         scores = []
@@ -426,7 +655,7 @@
 
         self.ensemble = _select_k(self.ensemble, order, k=self.red_size)
 
-    def _ptr_policy(self, predictions):
+    def _ptr_policy(self, predictions):
         """
         Selects the predictions made by models that have been trained on samples with a prevalence that is most similar
         to a first approximation of the test prevalence as made by all models in the ensemble.
@@ -437,7 +666,7 @@
         order = np.argsort(ptr_differences)
         return _select_k(predictions, order, k=self.red_size)
 
-    def _ds_policy_get_posteriors(self, data: LabelledCollection):
+    def _ds_policy_get_posteriors(self, data: LabelledCollection):
         """
         In the original article, there are some aspects regarding this method that are not mentioned. The paper says
         that the distribution of posterior probabilities from training and test examples is compared by means of the
@@ -468,7 +697,7 @@
 
         return posteriors, posteriors_generator
 
-    def _ds_policy(self, predictions, test):
+    def _ds_policy(self, predictions, test):
         test_posteriors = self.post_proba_fn(test)
         test_distribution = get_probability_distribution(test_posteriors)
         tr_distributions = [m[2] for m in self.ensemble]
@@ -477,7 +706,7 @@
         return _select_k(predictions, order, k=self.red_size)
 
     @property
-    def aggregative(self):
+    def aggregative(self):
         """
         Indicates that the quantifier is not aggregative.
 
@@ -486,7 +715,7 @@
         return False
 
     @property
-    def probabilistic(self):
+    def probabilistic(self):
         """
         Indicates that the quantifier is not probabilistic.
 
@@ -495,7 +724,10 @@
         return False

 
 
-
[docs]def get_probability_distribution(posterior_probabilities, bins=8):
+
+

+[docs]
+def get_probability_distribution(posterior_probabilities, bins=8):
     """
     Gets a histogram out of the posterior probabilities (only for the binary case).
 
@@ -509,14 +741,15 @@
     return distribution

 
 
-def _select_k(elements, order, k):
+
+def _select_k(elements, order, k):
     return [elements[idx] for idx in order[:k]]
 
 
-def _delayed_new_instance(args):
+def _delayed_new_instance(args):
     base_quantifier, data, val_split, prev, posteriors, keep_samples, verbose, sample_size = args
     if verbose:
-        print(f'\tfit-start for prev {F.strprev(prev)}, sample_size={sample_size}')
+        logging.getLogger(__name__).info(f'fit-start for prev {F.strprev(prev)}, sample_size={sample_size}')
     model = deepcopy(base_quantifier)
 
     if val_split is not None:
@@ -528,25 +761,25 @@
     sample = data.sampling_from_index(sample_index)
 
     if val_split is not None:
-        model.fit(sample, val_split=val_split)
+        model.fit(*sample.Xy, val_split=val_split)
     else:
-        model.fit(sample)
+        model.fit(*sample.Xy)
 
     tr_prevalence = sample.prevalence()
     tr_distribution = get_probability_distribution(posteriors[sample_index]) if (posteriors is not None) else None
 
     if verbose:
-        print(f'\t\--fit-ended for prev {F.strprev(prev)}')
+        logging.getLogger(__name__).info(f'fit-ended for prev {F.strprev(prev)}')
 
     return (model, tr_prevalence, tr_distribution, sample if keep_samples else None)
 
 
-def _delayed_quantify(args):
+def _delayed_quantify(args):
     quantifier, instances = args
-    return quantifier[0].quantify(instances)
+    return quantifier[0].predict(instances)
 
 
-def _draw_simplex(ndim, min_val, max_trials=100):
+def _draw_simplex(ndim, min_val, max_trials=100):
     """
     Returns a uniform sampling from the ndim-dimensional simplex but guarantees that all dimensions
     are >= min_class_prev (for min_val>0, this makes the sampling not truly uniform)
@@ -571,7 +804,7 @@
                              f'>= {min_val} is unlikely (it failed after {max_trials} trials)')
 
 
-def _instantiate_ensemble(classifier, base_quantifier_class, param_grid, optim, param_model_sel, **kwargs):
+def _instantiate_ensemble(classifier, base_quantifier_class, param_grid, optim, param_model_sel, **kwargs):
     if optim is None:
         base_quantifier = base_quantifier_class(classifier)
     elif optim in qp.error.CLASSIFICATION_ERROR:
@@ -590,7 +823,7 @@
     return Ensemble(base_quantifier, **kwargs)
 
 
-def _check_error(error):
+def _check_error(error):
     if error is None:
         return None
     if error in qp.error.QUANTIFICATION_ERROR or error in qp.error.CLASSIFICATION_ERROR:
@@ -602,7 +835,9 @@
                          f'the name of an error function in {qp.error.ERROR_NAMES}')
 
 
-
[docs]def ensembleFactory(classifier, base_quantifier_class, param_grid=None, optim=None, param_model_sel: dict = None,
+

+[docs]
+def ensembleFactory(classifier, base_quantifier_class, param_grid=None, optim=None, param_model_sel: dict = None,
                     **kwargs):
     """
     Ensemble factory. Provides a unified interface for instantiating ensembles that can be optimized (via model
@@ -653,7 +888,10 @@
     return _instantiate_ensemble(classifier, base_quantifier_class, param_grid, error, param_model_sel, **kwargs)

 
 
-
[docs]def ECC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def ECC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.CC` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -676,7 +914,10 @@
     return ensembleFactory(classifier, CC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.ACC` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -699,7 +940,10 @@
     return ensembleFactory(classifier, ACC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EPACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EPACC(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.PACC` quantifiers.
 
@@ -721,7 +965,10 @@
     return ensembleFactory(classifier, PACC, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EHDy(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EHDy(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.HDy` quantifiers, as used by
     `Pérez-Gállego et al., 2019 <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.
@@ -744,7 +991,10 @@
     return ensembleFactory(classifier, HDy, param_grid, optim, param_mod_sel, **kwargs)

 
 
-
[docs]def EEMQ(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
+
+

+[docs]
+def EEMQ(classifier, param_grid=None, optim=None, param_mod_sel=None, **kwargs):
     """
     Implements an ensemble of :class:`quapy.method.aggregative.EMQ` quantifiers.
 
@@ -764,33 +1014,212 @@
     """
 
     return ensembleFactory(classifier, EMQ, param_grid, optim, param_mod_sel, **kwargs)

+
+
+
+

+[docs]
+def merge(prev_predictions, merge_fun):
+    prev_predictions = np.asarray(prev_predictions)
+    if merge_fun == 'median':
+        prevalences = np.median(prev_predictions, axis=0)
+        prevalences = F.normalize_prevalence(prevalences, method='l1')
+    elif merge_fun == 'mean':
+        prevalences = np.mean(prev_predictions, axis=0)
+    else:
+        raise NotImplementedError(f'merge function {merge_fun} not implemented!')
+    return prevalences

+
+
+
+

+[docs]
+class SCMQ(AggregativeSoftQuantifier):
+
+    MERGE_FUNCTIONS = ['median', 'mean']
+
+    def __init__(self, classifier, quantifiers: List[AggregativeSoftQuantifier], merge_fun='median', val_split=5):
+        self.classifier = classifier
+        self.quantifiers = [deepcopy(q) for q in quantifiers]
+        assert merge_fun in self.MERGE_FUNCTIONS, f'unknown {merge_fun=}, valid ones are {self.MERGE_FUNCTIONS}'
+        self.merge_fun = merge_fun
+        self.val_split = val_split
+
+

+[docs]
+    def aggregation_fit(self, classif_predictions, labels):
+        for quantifier in self.quantifiers:
+            quantifier.classifier = self.classifier
+            quantifier.aggregation_fit(classif_predictions, labels)
+        return self

+
+
+

+[docs]
+    def aggregate(self, classif_predictions: np.ndarray):
+        prev_predictions = []
+        for quantifier_i in self.quantifiers:
+            prevalence_i = quantifier_i.aggregate(classif_predictions)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+

+[docs]
+class MCSQ(BaseQuantifier):
+    def __init__(self, classifiers, quantifier: AggregativeSoftQuantifier, merge_fun='median', val_split=5):
+        self.merge_fun = merge_fun
+        self.val_split = val_split
+        self.mcsqs = []
+        for classifier in classifiers:
+            quantifier = deepcopy(quantifier)
+            quantifier.classifier = classifier
+            self.mcsqs.append(quantifier)
+
+

+[docs]
+    def fit(self, data: LabelledCollection):
+        for q in self.mcsqs:
+            q.fit(data, val_split=self.val_split)
+        return self

+
+
+

+[docs]
+    def quantify(self, instances):
+        prev_predictions = []
+        for q in self.mcsqs:
+            prevalence_i = q.quantify(instances)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+

+[docs]
+class MCMQ(BaseQuantifier):
+    def __init__(self, classifiers, quantifiers: List[AggregativeSoftQuantifier], merge_fun='median', val_split=5):
+        self.merge_fun = merge_fun
+        self.scmqs = []
+        for classifier in classifiers:
+            self.scmqs.append(SCMQ(classifier, quantifiers, val_split=val_split))
+
+

+[docs]
+    def fit(self, data: LabelledCollection):
+        for q in self.scmqs:
+            q.fit(data)
+        return self

+
+
+

+[docs]
+    def quantify(self, instances):
+        prev_predictions = []
+        for q in self.scmqs:
+            prevalence_i = q.quantify(instances)
+            prev_predictions.append(prevalence_i)
+        return merge(prev_predictions, merge_fun=self.merge_fun)

+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/method/non_aggregative.html b/docs/build/html/_modules/quapy/method/non_aggregative.html index aeb5b96..aeda657 100644 --- a/docs/build/html/_modules/quapy/method/non_aggregative.html +++ b/docs/build/html/_modules/quapy/method/non_aggregative.html @@ -1,86 +1,397 @@ + - - - - - quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.method.non_aggregative — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.method.non_aggregative

-from typing import Union, Callable
-import numpy as np
+from itertools import product
+from tqdm import tqdm
+from typing import Union, Callable, Counter
+import numpy as np
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.utils import resample
+from sklearn.preprocessing import normalize
 
-from quapy.functional import get_divergence
-from quapy.data import LabelledCollection
-from quapy.method.base import BaseQuantifier, BinaryQuantifier
-import quapy.functional as F
+from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC
+from quapy.functional import get_divergence
+from quapy.method.base import BaseQuantifier, BinaryQuantifier
+from quapy.method._helper import _labels_to_indices
+from quapy.method._energy import _EnergyDistanceCore
+import quapy.functional as F
+from scipy.optimize import lsq_linear
+from scipy import sparse
+import quapy as qp
 
 
-
[docs]class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier):
+

+[docs]
+class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier):
     """
     The `Maximum Likelihood Prevalence Estimation` (MLPE) method is a lazy method that assumes there is no prior
     probability shift between training and test instances (put it other way, that the i.i.d. assumpion holds).
@@ -89,30 +400,41 @@
     any quantification method should beat.
     """
 
-    def __init__(self):
+    def __init__(self):
         self._classes_ = None
 
-
[docs]    def fit(self, data: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """
         Computes the training prevalence and stores it.
 
-        :param data: the training sample
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         :return: self
         """
-        self.estimated_prevalence = data.prevalence()
+        self._classes_ = F.classes_from_labels(labels=y)
+        self.estimated_prevalence = F.prevalence_from_labels(y, classes=self._classes_)
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """
         Ignores the input instances and returns, as the class prevalence estimantes, the training prevalence.
 
-        :param instances: array-like (ignored)
+        :param X: array-like (ignored)
         :return: the class prevalence seen during training
         """
-        return self.estimated_prevalence

+        return self.estimated_prevalence

+

 
 
-
[docs]class DMx(BaseQuantifier):
+
+

+[docs]
+class DMx(BaseQuantifier):
     """
     Generic Distribution Matching quantifier for binary or multiclass quantification based on the space of covariates.
     This implementation takes the number of bins, the divergence, and the possibility to work on CDF as hyperparameters.
@@ -122,18 +444,23 @@
         or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger
         Distance)
     :param cdf: whether to use CDF instead of PDF (default False)
+    :param search: string indicating the search strategy used to estimate the prevalence values.
+        Valid options are `optim_minimize` (default, works for binary and multiclass problems),
+        `linear_search` (binary only), and `ternary_search` (binary only)
     :param n_jobs: number of parallel workers (default None)
     """
 
-    def __init__(self, nbins=8, divergence: Union[str, Callable]='HD', cdf=False, search='optim_minimize', n_jobs=None):
+    def __init__(self, nbins=8, divergence: Union[str, Callable]='HD', cdf=False, search='optim_minimize', n_jobs=None):
         self.nbins = nbins
         self.divergence = divergence
         self.cdf = cdf
         self.search = search
         self.n_jobs = n_jobs
 
-
[docs]    @classmethod
-    def HDx(cls, n_jobs=None):
+

+[docs]
+    @classmethod
+    def HDx(cls, n_jobs=None):
         """
         `Hellinger Distance x <https://www.sciencedirect.com/science/article/pii/S0020025512004069>`_ (HDx).
         HDx is a method for training binary quantifiers, that models quantification as the problem of
@@ -149,15 +476,15 @@
         :return: an instance of this class setup to mimick the performance of the HDx as originally proposed by
             González-Castro, Alaiz-Rodríguez, Alegre (2013)
         """
-        from quapy.method.meta import MedianEstimator
+        from quapy.method.meta import MedianEstimator
 
         dmx = DMx(divergence='HD', cdf=False, search='linear_search')
         nbins = {'nbins': np.linspace(10, 110, 11, dtype=int)}
         hdx = MedianEstimator(base_quantifier=dmx, param_grid=nbins, n_jobs=n_jobs)
         return hdx

 
-    def __get_distributions(self, X):
 
+    def __get_distributions(self, X):
         histograms = []
         for feat_idx in range(self.nfeats):
             feature = X[:, feat_idx]
@@ -172,7 +499,9 @@
 
         return distributions
 
-
[docs]    def fit(self, data: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """
         Generates the validation distributions out of the training data (covariates).
         The validation distributions have shape `(n, nfeats, nbins)`, with `n` the number of classes, `nfeats`
@@ -181,46 +510,300 @@
         training data labelled with class `i`; while `dij = di[j]` is the discrete distribution for feature j in
         training data labelled with class `i`, and `dij[k]` is the fraction of instances with a value in the `k`-th bin.
 
-        :param data: the training set
+        :param X: array-like of shape `(n_samples, n_features)`, the training instances
+        :param y: array-like of shape `(n_samples,)`, the labels
         """
-        X, y = data.Xy
-
         self.nfeats = X.shape[1]
         self.feat_ranges = _get_features_range(X)
+        classes = np.unique(y)
+        y = _labels_to_indices(y, classes)
+        n_classes = len(classes)
 
         self.validation_distribution = np.asarray(
-            [self.__get_distributions(X[y==cat]) for cat in range(data.n_classes)]
+            [self.__get_distributions(X[y==cat]) for cat in range(n_classes)]
         )
 
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """
         Searches for the mixture model parameter (the sought prevalence values) that yields a validation distribution
         (the mixture) that best matches the test distribution, in terms of the divergence measure of choice.
         The matching is computed as the average dissimilarity (in terms of the dissimilarity measure of choice)
         between all feature-specific discrete distributions.
 
-        :param instances: instances in the sample
+        :param X: instances in the sample
         :return: a vector of class prevalence estimates
         """
 
-        assert instances.shape[1] == self.nfeats, f'wrong shape; expected {self.nfeats}, found {instances.shape[1]}'
+        assert X.shape[1] == self.nfeats, f'wrong shape; expected {self.nfeats}, found {X.shape[1]}'
 
-        test_distribution = self.__get_distributions(instances)
+        test_distribution = self.__get_distributions(X)
         divergence = get_divergence(self.divergence)
         n_classes, n_feats, nbins = self.validation_distribution.shape
-        def loss(prev):
+        def loss(prev):
             prev = np.expand_dims(prev, axis=0)
             mixture_distribution = (prev @ self.validation_distribution.reshape(n_classes,-1)).reshape(n_feats, -1)
             divs = [divergence(test_distribution[feat], mixture_distribution[feat]) for feat in range(n_feats)]
             return np.mean(divs)
 
-        return F.argmin_prevalence(loss, n_classes, method=self.search)

+        return F.argmin_prevalence(loss, n_classes, method=self.search)

+

 
 
 
-def _get_features_range(X):
+

+[docs]
+class EDx(_EnergyDistanceCore, BaseQuantifier):
+    """
+    Energy Distance x (EDx), a covariate-space distribution-matching
+    quantifier based on energy distance.
+
+    EDx is the classifier-free counterpart of :class:`quapy.method.aggregative.EDy`.
+    Instead of representing each class through posterior-probability vectors, it
+    represents each class by the cloud of raw feature vectors observed in the
+    training set and estimates the test prevalence vector by solving the same
+    energy-distance quadratic program directly in feature space.
+
+    This implementation works for binary and multiclass single-label
+    quantification and relies on the optional ``quadprog`` dependency. The
+    current QuaPy adaptation shares its numerical core with EDy and keeps
+    credit to the original implementation available in
+    `quantificationlib <https://github.com/AICGijon/quantificationlib>`_.
+
+    The formulation follows the same references as EDy, namely:
+
+    * Alberto Castaño, Laura Morán-Fernández, Jaime Alonso,
+      Verónica Bolón-Canedo, Amparo Alonso-Betanzos, and Juan José del Coz.
+      *An analysis of quantification methods based on matching distributions*.
+    * Hideko Kawakubo, Marthinus Christoffel du Plessis, and Masashi Sugiyama
+      (2016). *Computationally efficient class-prior estimation under class
+      balance change using energy distance*. IEICE Transactions on Information
+      and Systems, 99(1):176-186.
+
+    :param distance: distance used to compare feature vectors. Valid string
+        aliases are ``'manhattan'`` (default) and ``'euclidean'``; a custom
+        callable compatible with pairwise-distance signatures can also be used
+    :param n_jobs: number of parallel workers (default ``None``, meaning the
+        value is taken from the environment)
+    """
+
+    def __init__(self, distance: Union[str, Callable] = 'manhattan', n_jobs=None):
+        self.distance = distance
+        self.n_jobs = qp._get_njobs(n_jobs)
+        self.classes_ = None
+        self.n_features_in_ = None
+        self.train_distrib_ = None
+        self.train_n_cls_i_ = None
+        self.K_ = None
+        self.G_ = None
+        self.C_ = None
+        self.b_ = None
+        self.a_ = None
+
+

+[docs]
+    def fit(self, X, y):
+        """Fit class-conditional feature-space distributions from training data."""
+        self._check_ed_init_parameters()
+        labels = np.asarray(y)
+        self.classes_ = np.unique(labels)
+        self.n_features_in_ = X.shape[1]
+        train_distrib = [X[labels == class_] for class_ in self.classes_]
+        return self._fit_energy_model(train_distrib)

+
+
+

+[docs]
+    def predict(self, X):
+        """Estimate class prevalences for a test sample of raw instances."""
+        assert X.shape[1] == self.n_features_in_, (
+            f'wrong shape; expected {self.n_features_in_}, found {X.shape[1]}'
+        )
+        return self._predict_energy(X)

+

+
+
+
+

+[docs]
+class ReadMe(BaseQuantifier, WithConfidenceABC):
+    """
+    ReadMe is a non-aggregative quantification system proposed by
+    `Daniel Hopkins and Gary King, 2007. A method of automated nonparametric content analysis for
+    social science. American Journal of Political Science, 54(1):229–247.
+    <https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1540-5907.2009.00428.x>`_.
+    The idea is to estimate `Q(Y=i)` directly from:
+
+    :math:`Q(X)=\\sum_{i=1} Q(X|Y=i) Q(Y=i)`
+
+    via least-squares regression, i.e., without incurring the cost of computing posterior probabilities.
+    However, this poses a very difficult representation in which the vector `Q(X)` and the matrix `Q(X|Y=i)`
+    can be of very high dimensions. In order to render the problem tracktable, ReadMe performs bagging in
+    the feature space. ReadMe also combines bagging with bootstrap in order to derive confidence intervals
+    around point estimations.
+
+    We use the same default parameters as in the official
+    `R implementation <https://github.com/iqss-research/ReadMeV1/blob/master/R/prototype.R>`_.
+
+    :param prob_model: str ('naive', or 'full'), selects the modality in which the probabilities `Q(X)` and
+        `Q(X|Y)` are to be modelled. Options include "full", which corresponds to the original formulation of
+        ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which
+        `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where
+        `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes.
+        Of course, this approach is computationally prohibited for large `K`, so the authors advised against computing it
+        for matrices with `K>25` (although we recommend even smaller values of `K`). A much faster model is "naive", which
+        considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this
+        case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior).
+    :param bootstrap_trials: int, number of bootstrap trials (default 300)
+    :param bagging_trials: int, number of bagging trials (default 300)
+    :param bagging_range: int, number of features to keep for each bagging trial (default 15)
+    :param confidence_level: float, a value in (0,1) reflecting the desired confidence level (default 0.95)
+    :param region: str in 'intervals', 'ellipse', 'ellipse-clr'; indicates the preferred method for
+        defining the confidence region (see :class:`WithConfidenceABC`)
+    :param bonferroni: bool (default False), whether to apply Bonferroni correction when
+        `region='intervals'`. This parameter has no effect for ellipse-based regions.
+    :param random_state: int or None, allows replicability (default None)
+    :param verbose: bool, whether to display information during the process (default False)
+    """
+
+    MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION = 25
+    PROBABILISTIC_MODELS = ["naive", "full"]
+
+    def __init__(self,
+                 prob_model="full",
+                 bootstrap_trials=300,
+                 bagging_trials=300,
+                 bagging_range=15,
+                 confidence_level=0.95,
+                 region='intervals',
+                 bonferroni=False,
+                 random_state=None,
+                 verbose=False):
+        assert prob_model in ReadMe.PROBABILISTIC_MODELS, \
+            f'unknown {prob_model=}, valid ones are {ReadMe.PROBABILISTIC_MODELS=}'
+        self.prob_model = prob_model
+        self.bootstrap_trials = bootstrap_trials
+        self.bagging_trials = bagging_trials
+        self.bagging_range = bagging_range
+        self.confidence_level = confidence_level
+        self.region = region
+        self.bonferroni = bonferroni
+        self.random_state = random_state
+        self.verbose = verbose
+
+

+[docs]
+    def fit(self, X, y):
+        self._check_matrix(X)
+
+        self.rng = np.random.default_rng(self.random_state)
+        self.classes_ = np.unique(y)
+
+        Xsize = X.shape[0]
+
+        # Bootstrap loop
+        self.Xboots, self.yboots = [], []
+        for _ in range(self.bootstrap_trials):
+            idx = self.rng.choice(Xsize, size=Xsize, replace=True)
+            self.Xboots.append(X[idx])
+            self.yboots.append(y[idx])
+
+        return self

+
+
+

+[docs]
+    def predict_conf(self, X, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC):
+        self._check_matrix(X)
+        if confidence_level is None:
+            confidence_level = self.confidence_level
+
+        n_features = X.shape[1]
+        boots_prevalences = []
+        for Xboots, yboots in tqdm(
+                zip(self.Xboots, self.yboots),
+                desc='bootstrap predictions', total=self.bootstrap_trials, disable=not self.verbose
+        ):
+            bagging_estimates = []
+            for _ in range(self.bagging_trials):
+                feat_idx = self.rng.choice(n_features, size=self.bagging_range, replace=False)
+                Xboots_bagging = Xboots[:, feat_idx]
+                X_boots_bagging = X[:, feat_idx]
+                bagging_prev = self._quantify_iteration(Xboots_bagging, yboots, X_boots_bagging)
+                bagging_estimates.append(bagging_prev)
+
+            boots_prevalences.append(np.mean(bagging_estimates, axis=0))
+
+        conf = WithConfidenceABC.construct_region(boots_prevalences, confidence_level, method=self.region, bonferroni=self.bonferroni)
+        prev_estim = conf.point_estimate()
+
+        return prev_estim, conf

+
+
+

+[docs]
+    def predict(self, X):
+        prev_estim, _ = self.predict_conf(X)
+        return prev_estim

+
+
+    def _quantify_iteration(self, Xtr, ytr, Xte):
+        """Single ReadMe estimate."""
+        PX_given_Y = np.asarray([self._compute_P(Xtr[ytr == c]) for i,c in enumerate(self.classes_)])
+        PX = self._compute_P(Xte)
+
+        res = lsq_linear(A=PX_given_Y.T, b=PX, bounds=(0, 1))
+        pY = np.maximum(res.x, 0)
+        return pY / pY.sum()
+
+    def _check_matrix(self, X):
+        """the "full" model requires estimating empirical distributions; due to the high computational cost,
+                this function is only made available for binary matrices"""
+        if self.prob_model == 'full' and not self._is_binary_matrix(X):
+            raise ValueError('the empirical distribution can only be computed efficiently on binary matrices')
+
+    def _is_binary_matrix(self, X):
+        data = X.data if sparse.issparse(X) else X
+        return np.all((data == 0) | (data == 1))
+
+    def _compute_P(self, X):
+        if self.prob_model == 'naive':
+            return self._multinomial_distribution(X)
+        elif self.prob_model == 'full':
+            return self._empirical_distribution(X)
+        else:
+            raise ValueError(f'unknown {self.prob_model}; valid ones are {ReadMe.PROBABILISTIC_MODELS=}')
+
+    def _empirical_distribution(self, X):
+
+        if X.shape[1] > self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION:
+            raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions '
+                             f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}')
+
+        # we first convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5);
+        # this will speed up subsequent comparisons a lot
+        K = X.shape[1]
+        binary_powers = 1 << np.arange(K-1, -1, -1)     # (2^K, ..., 32, 16, 8, 4, 2, 1)
+        X_as_binary_numbers = X @ binary_powers         # e.g., [0 0 1 0 1] @ [16, 8, 4, 2, 1] = 5
+
+        # count occurrences and compute probs
+        counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float)
+        probs = counts / counts.sum()
+
+        return probs
+
+    def _multinomial_distribution(self, X):
+        PX = np.asarray(X.sum(axis=0))
+        PX = normalize(PX, norm='l1', axis=1)
+        return PX.ravel()

+
+
+
+def _get_features_range(X):
     feat_ranges = []
     ncols = X.shape[1]
     for col_idx in range(ncols):
@@ -233,34 +816,82 @@
 # aliases
 #---------------------------------------------------------------
 
+
+HDx = DMx.HDx
 DistributionMatchingX = DMx
+EnergyDistanceX = EDx
+HellingerDistanceX = HDx
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/model_selection.html b/docs/build/html/_modules/quapy/model_selection.html index 172c8f3..2123164 100644 --- a/docs/build/html/_modules/quapy/model_selection.html +++ b/docs/build/html/_modules/quapy/model_selection.html @@ -1,122 +1,437 @@ + - - - - - quapy.model_selection — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.model_selection — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.model_selection

-import itertools
-import signal
-from copy import deepcopy
-from enum import Enum
-from typing import Union, Callable
-from functools import wraps
+import itertools
+import logging
+import signal
+from copy import deepcopy
+from enum import Enum
+from typing import Union, Callable
+from functools import wraps
 
-import numpy as np
-from sklearn import clone
+import numpy as np
+from sklearn import clone
 
-import quapy as qp
-from quapy import evaluation
-from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol
-from quapy.data.base import LabelledCollection
-from quapy.method.aggregative import BaseQuantifier, AggregativeQuantifier
-from quapy.util import timeout
-from time import time
+import quapy as qp
+from quapy import evaluation
+from quapy.protocol import AbstractProtocol, OnLabelledCollectionProtocol
+from quapy.data.base import LabelledCollection
+from quapy.method.aggregative import BaseQuantifier, AggregativeQuantifier
+from quapy.util import timeout
+from time import time
 
 
-
[docs]class Status(Enum):
+

+[docs]
+class Status(Enum):
     SUCCESS = 1
     TIMEOUT = 2
     INVALID = 3
     ERROR = 4

 
 
-
[docs]class ConfigStatus:
-    def __init__(self, params, status, msg=''):
+
+

+[docs]
+class ConfigStatus:
+
+    def __init__(self, params, status, msg=''):
         self.params = params
         self.status = status
         self.msg = msg
 
-    def __str__(self):
+    def __str__(self):
         return f':params:{self.params} :status:{self.status} ' + self.msg
 
-    def __repr__(self):
+    def __repr__(self):
         return str(self)
 
-
[docs]    def success(self):
+

+[docs]
+    def success(self):
         return self.status == Status.SUCCESS

 
-
[docs]    def failed(self):
-        return self.status != Status.SUCCESS

+
+

+[docs]
+    def failed(self):
+        return self.status != Status.SUCCESS

+

 
 
-
[docs]class GridSearchQ(BaseQuantifier):
+
+

+[docs]
+class GridSearchQ(BaseQuantifier):
     """Grid Search optimization targeting a quantification-oriented metric.
 
     Optimizes the hyperparameters of a quantification method, based on an evaluation method and on an evaluation
@@ -139,7 +454,7 @@
     :param verbose: set to True to get information through the stdout
     """
 
-    def __init__(self,
+    def __init__(self,
                  model: BaseQuantifier,
                  param_grid: dict,
                  protocol: AbstractProtocol,
@@ -158,14 +473,14 @@
         self.n_jobs = qp._get_njobs(n_jobs)
         self.raise_errors = raise_errors
         self.verbose = verbose
-        self.__check_error(error)
+        self.__check_error_measure(error)
         assert isinstance(protocol, AbstractProtocol), 'unknown protocol'
 
-    def _sout(self, msg):
+    def _sout(self, msg):
         if self.verbose:
-            print(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}')
+            logging.getLogger(__name__).info(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}')
 
-    def __check_error(self, error):
+    def __check_error_measure(self, error):
         if error in qp.error.QUANTIFICATION_ERROR:
             self.error = error
         elif isinstance(error, str):
@@ -176,26 +491,27 @@
             raise ValueError(f'unexpected error type; must either be a callable function or a str representing\n'
                              f'the name of an error function in {qp.error.QUANTIFICATION_ERROR_NAMES}')
 
-    def _prepare_classifier(self, cls_params):
+    def _prepare_classifier(self, cls_params):
         model = deepcopy(self.model)
 
-        def job(cls_params):
+        def job(cls_params):
             model.set_params(**cls_params)
-            predictions = model.classifier_fit_predict(self._training)
+            predictions = model.classifier_fit_predict(self._training_X, self._training_y)
             return predictions
 
         predictions, status, took = self._error_handler(job, cls_params)
         self._sout(f'[classifier fit] hyperparams={cls_params} [took {took:.3f}s]')
         return model, predictions, status, took
 
-    def _prepare_aggregation(self, args):
+    def _prepare_aggregation(self, args):
         model, predictions, cls_took, cls_params, q_params = args
         model = deepcopy(model)
         params = {**cls_params, **q_params}
 
-        def job(q_params):
+        def job(q_params):
             model.set_params(**q_params)
-            model.aggregation_fit(predictions, self._training)
+            P, y = predictions
+            model.aggregation_fit(P, y)
             score = evaluation.evaluate(model, protocol=self.protocol, error_metric=self.error)
             return score
 
@@ -203,12 +519,12 @@
         self._print_status(params, score, status, aggr_took)
         return model, params, score, status, (cls_took+aggr_took)
 
-    def _prepare_nonaggr_model(self, params):
+    def _prepare_nonaggr_model(self, params):
         model = deepcopy(self.model)
 
-        def job(params):
+        def job(params):
             model.set_params(**params)
-            model.fit(self._training)
+            model.fit(self._training_X, self._training_y)
             score = evaluation.evaluate(model, protocol=self.protocol, error_metric=self.error)
             return score
 
@@ -216,7 +532,7 @@
         self._print_status(params, score, status, took)
         return model, params, score, status, took
 
-    def _break_down_fit(self):
+    def _break_down_fit(self):
         """
         Decides whether to break down the fit phase in two (classifier-fit followed by aggregation-fit).
         In order to do so, some conditions should be met: a) the quantifier is of type aggregative,
@@ -231,17 +547,19 @@
             return False
         return True
 
-    def _compute_scores_aggregative(self, training):
+    def _compute_scores_aggregative(self, X, y):
         # break down the set of hyperparameters into two: classifier-specific, quantifier-specific
         cls_configs, q_configs = group_params(self.param_grid)
 
         # train all classifiers and get the predictions
-        self._training = training
+        self._training_X = X
+        self._training_y = y
         cls_outs = qp.util.parallel(
             self._prepare_classifier,
             cls_configs,
             seed=qp.environ.get('_R_SEED', None),
-            n_jobs=self.n_jobs
+            n_jobs=self.n_jobs,
+            asarray=False
         )
 
         # filter out classifier configurations that yielded any error
@@ -266,9 +584,10 @@
 
         return aggr_outs
 
-    def _compute_scores_nonaggregative(self, training):
+    def _compute_scores_nonaggregative(self, X, y):
         configs = expand_grid(self.param_grid)
-        self._training = training
+        self._training_X = X
+        self._training_y = y
         scores = qp.util.parallel(
             self._prepare_nonaggr_model,
             configs,
@@ -277,17 +596,20 @@
         )
         return scores
 
-    def _print_status(self, params, score, status, took):
+    def _print_status(self, params, score, status, took):
         if status.success():
             self._sout(f'hyperparams=[{params}]\t got {self.error.__name__} = {score:.5f} [took {took:.3f}s]')
         else:
             self._sout(f'error={status}')
 
-
[docs]    def fit(self, training: LabelledCollection):
+

+[docs]
+    def fit(self, X, y):
         """ Learning routine. Fits methods with all combinations of hyperparameters and selects the one minimizing
             the error metric.
 
-        :param training: the training set on which to optimize the hyperparameters
+        :param X: array-like, training covariates
+        :param y: array-like, labels of training data
         :return: self
         """
 
@@ -303,9 +625,9 @@
 
         self._sout(f'starting model selection with n_jobs={self.n_jobs}')
         if self._break_down_fit():
-            results = self._compute_scores_aggregative(training)
+            results = self._compute_scores_aggregative(X, y)
         else:
-            results = self._compute_scores_nonaggregative(training)
+            results = self._compute_scores_nonaggregative(X, y)
 
         self.param_scores_ = {}
         self.best_score_ = None
@@ -320,13 +642,13 @@
                 self.param_scores_[str(params)] = status.status
                 self.error_collector.append(status)
 
-        tend = time()-tinit
+        self.fit_time_ = time()-tinit
 
         if self.best_score_ is None:
             raise ValueError('no combination of hyperparameters seemed to work')
 
         self._sout(f'optimization finished: best params {self.best_params_} (score={self.best_score_:.5f}) '
-                   f'[took {tend:.4f}s]')
+                   f'[took {self.fit_time_:.4f}s]')
 
         no_errors = len(self.error_collector)
         if no_errors>0:
@@ -338,7 +660,10 @@
             if isinstance(self.protocol, OnLabelledCollectionProtocol):
                 tinit = time()
                 self._sout(f'refitting on the whole development set')
-                self.best_model_.fit(training + self.protocol.get_labelled_collection())
+                validation_collection = self.protocol.get_labelled_collection()
+                training_collection = LabelledCollection(X, y, classes=validation_collection.classes)
+                devel_collection = training_collection + validation_collection
+                self.best_model_.fit(*devel_collection.Xy)
                 tend = time() - tinit
                 self.refit_time_ = tend
             else:
@@ -347,24 +672,33 @@
 
         return self

 
-
[docs]    def quantify(self, instances):
+
+

+[docs]
+    def predict(self, X):
         """Estimate class prevalence values using the best model found after calling the :meth:`fit` method.
 
-        :param instances: sample contanining the instances
+        :param X: sample contanining the instances
         :return: a ndarray of shape `(n_classes)` with class prevalence estimates as according to the best model found
             by the model selection process.
         """
         assert hasattr(self, 'best_model_'), 'quantify called before fit'
-        return self.best_model().quantify(instances)

+        return self.best_model().predict(X)

 
-
[docs]    def set_params(self, **parameters):
+
+

+[docs]
+    def set_params(self, **parameters):
         """Sets the hyper-parameters to explore.
 
         :param parameters: a dictionary with keys the parameter names and values the list of values to explore
         """
         self.param_grid = parameters

 
-
[docs]    def get_params(self, deep=True):
+
+

+[docs]
+    def get_params(self, deep=True):
         """Returns the dictionary of hyper-parameters to explore (`param_grid`)
 
         :param deep: Unused
@@ -372,7 +706,10 @@
         """
         return self.param_grid

 
-
[docs]    def best_model(self):
+
+

+[docs]
+    def best_model(self):
         """
         Returns the best model found after calling the :meth:`fit` method, i.e., the one trained on the combination
         of hyper-parameters that minimized the error function.
@@ -383,7 +720,8 @@
             return self.best_model_
         raise ValueError('best_model called before fit')

 
-    def _error_handler(self, func, params):
+
+    def _error_handler(self, func, params):
         """
         Endorses one job with two returned values: the status, and the time of execution
 
@@ -396,11 +734,11 @@
 
         output = None
 
-        def _handle(status, exception):
+        def _handle(status, exception):
             if self.raise_errors:
                 raise exception
             else:
-                return ConfigStatus(params, status)
+                return ConfigStatus(params, status, msg=str(exception))
 
         try:
             with timeout(self.timeout):
@@ -421,7 +759,10 @@
         return output, status, took

 
 
-
[docs]def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfolds=3, random_state=0):
+
+

+[docs]
+def cross_val_predict(quantifier: BaseQuantifier, data: LabelledCollection, nfolds=3, random_state=0):
     """
     Akin to `scikit-learn's cross_val_predict <https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_val_predict.html>`_
     but for quantification.
@@ -436,15 +777,18 @@
     total_prev = np.zeros(shape=data.n_classes)
 
     for train, test in data.kFCV(nfolds=nfolds, random_state=random_state):
-        quantifier.fit(train)
-        fold_prev = quantifier.quantify(test.X)
+        quantifier.fit(*train.Xy)
+        fold_prev = quantifier.predict(test.X)
         rel_size = 1. * len(test) / len(data)
         total_prev += fold_prev*rel_size
 
     return total_prev

 
 
-
[docs]def expand_grid(param_grid: dict):
+
+

+[docs]
+def expand_grid(param_grid: dict):
     """
     Expands a param_grid dictionary as a list of configurations.
     Example:
@@ -463,7 +807,10 @@
     return configs

 
 
-
[docs]def group_params(param_grid: dict):
+
+

+[docs]
+def group_params(param_grid: dict):
     """
     Partitions a param_grid dictionary as two lists of configurations, one for the classifier-specific
     hyper-parameters, and another for que quantifier-specific hyper-parameters
@@ -484,33 +831,78 @@
 
     return classifier_configs, quantifier_configs

 
+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/plot.html b/docs/build/html/_modules/quapy/plot.html index 79179a1..10c742c 100644 --- a/docs/build/html/_modules/quapy/plot.html +++ b/docs/build/html/_modules/quapy/plot.html @@ -1,85 +1,388 @@ + - - - - - quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.plot — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ +

Source code for quapy.plot

+from collections import defaultdict
+import math
+
+from matplotlib import cm
+import matplotlib.colors as mcolors
+import matplotlib.pyplot as plt
+from matplotlib.colors import LinearSegmentedColormap, ListedColormap
+from matplotlib.pyplot import get_cmap
+from matplotlib.ticker import ScalarFormatter
+import numpy as np
+from scipy.stats import ttest_ind_from_stats
+
+import quapy as qp
 
 plt.rcParams['figure.figsize'] = [10, 6]
 plt.rcParams['figure.dpi'] = 200
@@ -88,7 +391,7 @@
 
 

 [docs]
-def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
+def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
                     train_prev=None, savepath=None, method_order=None):
     """
     The diagonal plot displays the predicted prevalence values (along the y-axis) as a function of the true prevalence
@@ -97,21 +400,29 @@
     indicating which class is to be taken as the positive class. (For multiclass quantification problems, other plots
     like the :meth:`error_by_drift` might be preferable though).
 
+    The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same
+    length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and
+    `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on
+    different datasets can be used, in which case the method name can appear more than once in `method_names`.
+
     :param method_names: array-like with the method names for each experiment
-    :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
-        each experiment
-    :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
-        for each experiment
-    :param pos_class: index of the positive class
-    :param title: the title to be displayed in the plot
-    :param show_std: whether or not to show standard deviations (represented by color bands). This might be inconvenient
+    :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components.
+    :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of
+        shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in
+        `true_prevs`.
+    :param pos_class: index of the positive class (default 1)
+    :param title: the title to be displayed in the plot (default None)
+    :param show_std: whether to show standard deviations (represented by color bands). This might be inconvenient
         for cases in which many methods are compared, or when the standard deviations are high -- default True)
-    :param legend: whether or not to display the leyend (default True)
-    :param train_prev: if indicated (default is None), the training prevalence (for the positive class) is hightlighted
-        in the plot. This is convenient when all the experiments have been conducted in the same dataset.
+    :param legend: whether to display the legend (default True)
+    :param train_prev: if indicated (default is None), the training prevalence (for the positive class) is highlighted
+        in the plot. This is convenient when all the experiments have been conducted in the same dataset, or in
+        datasets with the same training prevalence.
     :param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
     :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
         listed in the legend and associated with matplotlib colors).
+    :return: returns (fig, ax) matplotlib objects for eventual customisation
     """
     fig, ax = plt.subplots()
     ax.set_aspect('equal')
@@ -152,31 +463,34 @@
 
     if legend:
         ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
-        # box = ax.get_position()
-        # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
-        # ax.legend(loc='lower center',
-        #           bbox_to_anchor=(1, -0.5),
-        #           ncol=(len(method_names)+1)//2)
 
-    _save_or_show(savepath)

+    _save_or_show(savepath)
+    return fig, ax
[docs] -def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None): +def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None): """ Box-plots displaying the global bias (i.e., signed error computed as the estimated value minus the true value) for each quantification method with respect to a given positive class. + The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same + length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and + `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on + different datasets can be used, in which case the method name can appear more than once in `method_names`. + :param method_names: array-like with the method names for each experiment - :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for - each experiment - :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) - for each experiment + :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components. + :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in + `true_prevs`. :param pos_class: index of the positive class - :param title: the title to be displayed in the plot + :param title: the title to be displayed in the plot (default None) :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + :return: returns (fig, ax) matplotlib objects for eventual customisation """ method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs) @@ -195,32 +509,41 @@ plt.xticks(rotation=45) ax.set(ylabel='error bias', title=title) - _save_or_show(savepath)
+ _save_or_show(savepath) + + return fig, ax
[docs] -def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10, +def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10, vertical_xticks=False, legend=True, savepath=None): """ Box-plots displaying the local bias (i.e., signed error computed as the estimated value minus the true value) - for different bins of (true) prevalence of the positive classs, for each quantification method. + for different bins of (true) prevalence of the positive class, for each quantification method. + + The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same + length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and + `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on + different datasets can be used, in which case the method name can appear more than once in `method_names`. :param method_names: array-like with the method names for each experiment - :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for - each experiment - :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) - for each experiment + :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components. + :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in + `true_prevs`. :param pos_class: index of the positive class - :param title: the title to be displayed in the plot - :param nbins: number of bins + :param title: the title to be displayed in the plot (default None) + :param nbins: number of bins (default 5) :param colormap: the matplotlib colormap to use (default cm.tab10) :param vertical_xticks: whether or not to add secondary grid (default is False) :param legend: whether or not to display the legend (default is True) :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + :return: returns (fig, ax) matplotlib objects for eventual customisation """ - from pylab import boxplot, plot, setp + from pylab import boxplot, plot, setp fig, ax = plt.subplots() ax.grid() @@ -288,18 +611,20 @@ ax.set(xlabel='prevalence', ylabel='error bias', title=title) ax.set_xlim(0, 1) - _save_or_show(savepath)
+ _save_or_show(savepath) + + return fig, ax
[docs] -def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, +def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, error_name='ae', show_std=False, show_density=True, show_legend=True, logscale=False, - title=f'Quantification error as a function of distribution shift', + title=None, vlines=None, method_order=None, savepath=None): @@ -310,11 +635,17 @@ fare in different regions of the prior probability shift spectrum (e.g., in the low-shift regime vs. in the high-shift regime). + The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same + length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and + `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on + different datasets can be used, in which case the method name can appear more than once in `method_names`. + :param method_names: array-like with the method names for each experiment - :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for - each experiment - :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) - for each experiment + :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components. + :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in + `true_prevs`. :param tr_prevs: training prevalence of each experiment :param n_bins: number of bins in which the y-axis is to be divided (default is 20) :param error_name: a string representing the name of an error function (as defined in `quapy.error`, default is "ae") @@ -322,12 +653,13 @@ :param show_density: whether or not to display the distribution of experiments for each bin (default is True) :param show_density: whether or not to display the legend of the chart (default is True) :param logscale: whether or not to log-scale the y-error measure (default is False) - :param title: title of the plot (default is "Quantification error as a function of distribution shift") + :param title: title of the plot (default is None) :param vlines: array-like list of values (default is None). If indicated, highlights some regions of the space using vertical dotted lines. :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e., listed in the legend and associated with matplotlib colors). :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + :return: returns (fig, ax) matplotlib objects for eventual customisation """ fig, ax = plt.subplots() @@ -336,14 +668,14 @@ x_error = qp.error.ae y_error = getattr(qp.error, error_name) + if method_order is None: + method_order = [] + # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to # x_error function) and 'y' is the estim-test shift (computed as according to y_error) data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order) - if method_order is None: - method_order = method_names - _set_colors(ax, n_methods=len(method_order)) bins = np.linspace(0, 1, n_bins+1) @@ -396,11 +728,11 @@ ax2.spines['right'].set_color('g') ax2.tick_params(axis='y', colors='g') - ax.set(xlabel=f'Distribution shift between training set and test sample', - ylabel=f'{error_name.upper()} (true distribution, predicted distribution)', + ax.set(xlabel=f'Prior shift between training set and test sample', + ylabel=f'{error_name.upper()} (true prev, predicted prev)', title=title) - box = ax.get_position() - ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) + # box = ax.get_position() + # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) if vlines: for vline in vlines: ax.axvline(vline, 0, 1, linestyle='--', color='k') @@ -410,19 +742,20 @@ #nice scale for the logaritmic axis ax.set_ylim(0,10 ** math.ceil(math.log10(max_y))) - if show_legend: - fig.legend(loc='lower center', + fig.legend(loc='center left', bbox_to_anchor=(1, 0.5), - ncol=(len(method_names)+1)//2) - - _save_or_show(savepath)
+ ncol=1) + + _save_or_show(savepath) + + return fig, ax
[docs] -def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, +def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, binning='isomerous', x_error='ae', y_error='ae', ttest_alpha=0.005, tail_density_threshold=0.005, method_order=None, @@ -436,11 +769,17 @@ plot is displayed on top, that displays the distribution of experiments for each bin (when binning="isometric") or the percentiles points of the distribution (when binning="isomerous"). + The format convention is as follows: `method_names`, `true_prevs`, and `estim_prevs` are array-like of the same + length, with the ith element describing the output of an independent experiment. The elements of `true_prevs`, and + `estim_prevs` are `ndarrays` with coherent shape for the same experiment. Experiments for the same method on + different datasets can be used, in which case the method name can appear more than once in `method_names`. + :param method_names: array-like with the method names for each experiment - :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for - each experiment - :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) - for each experiment + :param true_prevs: array-like with the true prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components. + :param estim_prevs: array-like with the estimated prevalence values for each experiment. Each entry is a ndarray of + shape `(n_samples, n_classes)` components and `n_samples` must coincide with the corresponding entry in + `true_prevs`. :param tr_prevs: training prevalence of each experiment :param n_bins: number of bins in which the y-axis is to be divided (default is 20) :param binning: type of binning, either "isomerous" (default) or "isometric" @@ -457,13 +796,16 @@ :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e., listed in the legend and associated with matplotlib colors). :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. - :return: + :return: returns (fig, ax) matplotlib objects for eventual customisation """ assert binning in ['isomerous', 'isometric'], 'unknown binning type; valid types are "isomerous" and "isometric"' x_error = getattr(qp.error, x_error) y_error = getattr(qp.error, y_error) + if method_order is None: + method_order = [] + # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to # x_error function) and 'y' is the estim-test shift (computed as according to y_error) @@ -527,7 +869,6 @@ best_bucket_methods.append(method_order[method_index]) best_methods.append(best_bucket_methods) salient_methods.update(best_bucket_methods) - print(best_bucket_methods) if binning=='isomerous': fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods))) @@ -602,11 +943,133 @@ ax.get_xaxis().set_visible(False) plt.subplots_adjust(wspace=0, hspace=0) - _save_or_show(savepath)
+ _save_or_show(savepath) + + return fig, ax
-def _merge(method_names, true_prevs, estim_prevs): +
+[docs] +def plot_simplex( + point_layers=None, + region_layers=None, + density_function=None, + density_color='#1f77b4', + density_alpha=1.0, + resolution=400, + class_names=None, + title=None, + show_legend=True, + legend_loc='lower center', + legend_bbox_to_anchor=(0.5, -0.08), + legend_ncol=2, + figsize=(6.8, 6.2), + class_name_fontsize=10, + title_fontsize=11, + legend_fontsize=9, + ax=None, + savepath=None): + """ + Plots data on the ternary simplex for three-class quantification problems. + + This utility is convenient for visualising prevalence vectors, posterior triplets, + confidence regions, or any other points that lie on the 2-dimensional probability + simplex. The plot can combine three optional layer types: + + * `point_layers`: scatter layers for one or more prevalence clouds or reference points + * `region_layers`: shaded regions defined by callables on prevalence vectors + * `density_function`: a scalar function evaluated on the simplex and rendered as a heatmap + + Each entry in `point_layers` is a dictionary with a mandatory `points` field + containing an array-like of shape `(n_points, 3)` or `(3,)`. Optional fields are + `label` for the legend and `style` for matplotlib scatter keyword arguments. + + Each entry in `region_layers` is a dictionary with a mandatory `fn` field containing + a callable that receives prevalence vectors and returns region-membership scores. + Optional fields are `label`, `color`, and `alpha`. + + :param point_layers: optional list of point-layer dictionaries + :param region_layers: optional list of region-layer dictionaries + :param density_function: optional callable receiving prevalence vectors and returning + scalar values + :param density_color: color used for the density heatmap + :param density_alpha: opacity for the density heatmap + :param resolution: number of grid steps per axis used for rendering regions and densities + :param class_names: optional list or tuple with the three class names + :param title: optional plot title + :param show_legend: whether to display the legend + :param legend_loc: location string passed to matplotlib for the legend + :param legend_bbox_to_anchor: optional legend anchor box + :param legend_ncol: number of legend columns + :param figsize: figure size used when `ax` is not provided + :param class_name_fontsize: fontsize used for simplex vertex labels + :param title_fontsize: fontsize used for the optional title + :param legend_fontsize: fontsize used for the legend + :param ax: optional matplotlib axes object; if not provided, a new figure is created + :param savepath: path where to save the plot; if not indicated, the plot is shown when + `ax` is not provided + :return: returns `(fig, ax)` matplotlib objects for eventual customisation + """ + if class_names is None: + class_names = ('Y=1', 'Y=2', 'Y=3') + if len(class_names) != 3: + raise ValueError(f'expected exactly 3 class names, got {len(class_names)}') + + own_figure = ax is None + if own_figure: + fig, ax = plt.subplots(figsize=figsize) + else: + fig = ax.figure + + if density_function is not None: + _plot_simplex_density(ax, density_function, resolution, density_color, density_alpha) + + if region_layers: + _plot_simplex_regions(ax, region_layers, resolution) + + if point_layers: + _plot_simplex_points(ax, point_layers) + + simplex_ymax = np.sqrt(3) / 2 + triangle = np.array([ + [0.0, 0.0], + [1.0, 0.0], + [0.5, simplex_ymax], + [0.0, 0.0], + ]) + ax.plot(triangle[:, 0], triangle[:, 1], color='black') + + ax.text(-0.05, -0.05, class_names[0], ha='right', va='top', fontsize=class_name_fontsize) + ax.text(1.05, -0.05, class_names[1], ha='left', va='top', fontsize=class_name_fontsize) + ax.text(0.5, simplex_ymax + 0.05, class_names[2], ha='center', va='bottom', fontsize=class_name_fontsize) + + if title is not None: + ax.set_title(title, fontsize=title_fontsize) + + ax.set_aspect('equal') + ax.set_xlim(-0.1, 1.1) + ax.set_ylim(-0.1, simplex_ymax + 0.1) + ax.axis('off') + + if show_legend: + _, labels = ax.get_legend_handles_labels() + if labels: + ax.legend(loc=legend_loc, bbox_to_anchor=legend_bbox_to_anchor, ncol=legend_ncol, fontsize=legend_fontsize, frameon=False) + + fig.tight_layout(pad=0.6) + + if savepath is not None: + qp.util.create_parent_dir(savepath) + fig.savefig(savepath, bbox_inches='tight') + elif own_figure: + plt.show() + + return fig, ax
+ + + +def _merge(method_names, true_prevs, estim_prevs): ndims = true_prevs[0].shape[1] data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))}) method_order=[] @@ -620,13 +1083,14 @@ return method_order, true_prevs_, estim_prevs_ -def _set_colors(ax, n_methods): +def _set_colors(ax, n_methods): NUM_COLORS = n_methods - cm = plt.get_cmap('tab20') - ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)]) + if NUM_COLORS>10: + cm = plt.get_cmap('tab20') + ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)]) -def _save_or_show(savepath): +def _save_or_show(savepath): # if savepath is specified, then saves the plot in that path; otherwise the plot is shown if savepath is not None: qp.util.create_parent_dir(savepath) @@ -636,7 +1100,7 @@ plt.show() -def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order): +def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order): data = defaultdict(lambda: {'x': np.empty(shape=(0)), 'y': np.empty(shape=(0))}) if method_order is None: @@ -655,33 +1119,203 @@ method_order.append(method) return data + + +def _simplex_to_cartesian(prevalences): + prevalences = np.asarray(prevalences, dtype=float) + prevalences = np.atleast_2d(prevalences) + if prevalences.shape[1] != 3: + raise ValueError(f'plot_simplex expects prevalence vectors of shape (_, 3); found {prevalences.shape}') + x = prevalences[:, 1] + 0.5 * prevalences[:, 2] + y = prevalences[:, 2] * (np.sqrt(3) / 2) + return x, y + + +def _barycentric_from_xy(x, y): + p3 = 2 * y / np.sqrt(3) + p2 = x - 0.5 * p3 + p1 = 1 - p2 - p3 + return np.stack([p1, p2, p3], axis=-1) + + +def _simplex_mesh(resolution): + simplex_ymax = np.sqrt(3) / 2 + xs = np.linspace(0, 1, resolution) + ys = np.linspace(0, simplex_ymax, resolution) + grid_x, grid_y = np.meshgrid(xs, ys) + pts_bary = _barycentric_from_xy(grid_x, grid_y) + mask = np.all(pts_bary >= 0, axis=-1) + return xs, ys, pts_bary, mask + + +def _evaluate_simplex_function(function, points): + points = np.asarray(points, dtype=float) + try: + values = np.asarray(function(points), dtype=float) + if values.shape == (points.shape[0],): + return values + if values.shape == points.shape[:-1]: + return values.reshape(-1) + except Exception: + pass + return np.asarray([function(point) for point in points], dtype=float) + + +def _region_colormap(color='blue', alpha=0.35): + return ListedColormap([ + (1.0, 1.0, 1.0, 0.0), + (*mcolors.to_rgb(color), alpha), + ]) + + +def _plot_simplex_points(ax, point_layers): + for layer in point_layers: + points = np.asarray(layer['points'], dtype=float) + style = {'s': 25, 'alpha': 0.8} + style.update(layer.get('style', {})) + ax.scatter(*_simplex_to_cartesian(points), label=layer.get('label'), **style) + + +def _plot_simplex_regions(ax, region_layers, resolution): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + + for layer in region_layers: + mask = np.zeros(simplex_mask.shape, dtype=float) + values = _evaluate_simplex_function(layer['fn'], valid_points) + mask[simplex_mask] = values + ax.pcolormesh( + xs, + ys, + mask, + shading='auto', + cmap=_region_colormap(layer.get('color', 'blue'), layer.get('alpha', 0.35)), + ) + if layer.get('label') is not None: + ax.scatter([], [], color=layer.get('color', 'blue'), alpha=layer.get('alpha', 0.35), label=layer['label']) + + +def _plot_simplex_density(ax, density_function, resolution, color, alpha): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + density = np.full(simplex_mask.shape, np.nan, dtype=float) + values = _evaluate_simplex_function(density_function, valid_points) + min_v, max_v = np.min(values), np.max(values) + if max_v > min_v: + values = (values - min_v) / (max_v - min_v) + density[simplex_mask] = values + + cmap = LinearSegmentedColormap.from_list('simplex_density', ['white', color]) + ax.pcolormesh(xs, ys, density, shading='auto', cmap=cmap, alpha=alpha) + + +
+[docs] +def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None): + posteriors = prob_classifier.predict_proba(X) + assert posteriors.ndim==2, 'calibration plot only works for binary problems' + posteriors = posteriors[:,1] + pred_y = posteriors>=0.5 + bins = np.linspace(0, 1, nbins + 1) + binned_values = np.digitize(posteriors, bins, right=False) + correct = pred_y == y + bin_centers = (bins[:-1] + bins[1:]) / 2 + bins_names = np.arange(nbins) + y_axis = [correct[binned_values==bin].mean() for bin in bins_names] + y_axis = [v if not np.isnan(v) else 0 for v in y_axis] + # Crear el gráfico de barras + plt.bar(bin_centers, y_axis, width=bins[1]-bins[0], edgecolor='black', alpha=0.7) + + # Etiquetas y título + plt.xlabel("Bin") + plt.ylabel("Value") + plt.title("Bar plot of calculated values per bin") + plt.xticks(bin_centers, [f"{b:.2f}" for b in bin_centers], rotation=45) + + # Mostrar el gráfico + plt.tight_layout() + plt.show()
+ + +if __name__ == '__main__': + import quapy as qp + from sklearn.linear_model import LogisticRegression + data = qp.datasets.fetch_UCIBinaryDataset(qp.datasets.UCI_BINARY_DATASETS[6]) + train, test = data.train_test + classifier = LogisticRegression() + classifier.fit(*train.Xy) + calibration_plot(classifier, *test.Xy)
- + + + + + + + + + + + + + - + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/protocol.html b/docs/build/html/_modules/quapy/protocol.html index 7d96338..f919481 100644 --- a/docs/build/html/_modules/quapy/protocol.html +++ b/docs/build/html/_modules/quapy/protocol.html @@ -1,95 +1,400 @@ + - - - - - quapy.protocol — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.protocol — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.protocol

-from copy import deepcopy
-import quapy as qp
-import numpy as np
-import itertools
-from contextlib import ExitStack
-from abc import ABCMeta, abstractmethod
-from quapy.data import LabelledCollection
-import quapy.functional as F
-from os.path import exists
-from glob import glob
+from copy import deepcopy
+from typing import Iterable
+
+import quapy as qp
+import numpy as np
+import itertools
+from contextlib import ExitStack
+from abc import ABCMeta, abstractmethod
+from quapy.data import LabelledCollection
+import quapy.functional as F
+from os.path import exists
+from glob import glob
+from collections.abc import Iterable
+from numbers import Number
 
 
-
[docs]class AbstractProtocol(metaclass=ABCMeta):
+

+[docs]
+class AbstractProtocol(metaclass=ABCMeta):
     """
     Abstract parent class for sample generation protocols.
     """
 
     @abstractmethod
-    def __call__(self):
+    def __call__(self):
         """
         Implements the protocol. Yields one sample at a time along with its prevalence
 
@@ -98,25 +403,31 @@
         """
         ...
 
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Indicates the total number of samples that the protocol generates.
 
         :return: The number of samples to generate if known, or `None` otherwise.
         """
-        return None

+        return None

+

 
 
-
[docs]class IterateProtocol(AbstractProtocol):
+
+

+[docs]
+class IterateProtocol(AbstractProtocol):
     """
     A very simple protocol which simply iterates over a list of previously generated samples
 
     :param samples: a list of :class:`quapy.data.base.LabelledCollection`
     """
-    def __init__(self, samples: [LabelledCollection]):
+    def __init__(self, samples: [LabelledCollection]):
         self.samples = samples
 
-    def __call__(self):
+    def __call__(self):
         """
         Yields one sample from the initial list at a time
 
@@ -126,16 +437,58 @@
         for sample in self.samples:
             yield sample.Xp
 
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples in this protocol
 
         :return: int
         """
-        return len(self.samples)

+        return len(self.samples)

+

 
 
-
[docs]class AbstractStochasticSeededProtocol(AbstractProtocol):
+
+

+[docs]
+class ProtocolFromIndex(AbstractProtocol):
+    """
+    A protocol from a list of indexes
+
+    :param data: a :class:`quapy.data.base.LabelledCollection`
+    :param indexes: a list of indexes
+    """
+    def __init__(self, data: LabelledCollection, indexes: Iterable):
+        self.data = data
+        self.indexes = indexes
+
+    def __call__(self):
+        """
+        Yields one sample at a time extracted using the indexes
+
+        :return: yields a tuple `(sample, prev) at a time, where `sample` is a set of instances
+            and in which `prev` is an `nd.array` with the class prevalence values
+        """
+        for index in self.indexes:
+            yield self.data.sampling_from_index(index).Xp
+
+

+[docs]
+    def total(self):
+        """
+        Returns the number of samples in this protocol
+
+        :return: int
+        """
+        return len(self.indexes)

+

+
+
+
+

+[docs]
+class AbstractStochasticSeededProtocol(AbstractProtocol):
     """
     An `AbstractStochasticSeededProtocol` is a protocol that generates, via any random procedure (e.g.,
     via random sampling), sequences of :class:`quapy.data.base.LabelledCollection` samples.
@@ -152,19 +505,21 @@
 
     _random_state = -1  # means "not set"
 
-    def __init__(self, random_state=0):
+    def __init__(self, random_state=0):
         self.random_state = random_state
 
     @property
-    def random_state(self):
+    def random_state(self):
         return self._random_state
 
     @random_state.setter
-    def random_state(self, random_state):
+    def random_state(self, random_state):
         self._random_state = random_state
 
-
[docs]    @abstractmethod
-    def samples_parameters(self):
+

+[docs]
+    @abstractmethod
+    def samples_parameters(self):
         """
         This function has to return all the necessary parameters to replicate the samples
 
@@ -172,8 +527,11 @@
         """
         ...

 
-
[docs]    @abstractmethod
-    def sample(self, params):
+
+

+[docs]
+    @abstractmethod
+    def sample(self, params):
         """
         Extract one sample determined by the given parameters
 
@@ -182,7 +540,8 @@
         """
         ...

 
-    def __call__(self):
+
+    def __call__(self):
         """
         Yields one sample at a time. The type of object returned depends on the `collator` function. The
         default behaviour returns tuples of the form `(sample, prevalence)`.
@@ -197,9 +556,11 @@
             if self.random_state is not None:
                 stack.enter_context(qp.util.temp_seed(self.random_state))
             for params in self.samples_parameters():
-                yield self.collator(self.sample(params))
+                yield self.collator(self.sample(params), params)
 
-
[docs]    def collator(self, sample, *args):
+

+[docs]
+    def collator(self, sample, params):
         """
         The collator prepares the sample to accommodate the desired output format before returning the output.
         This collator simply returns the sample as it is. Classes inheriting from this abstract class can
@@ -209,17 +570,23 @@
         :param args: additional arguments
         :return: the sample adhering to a desired output format (in this case, the sample is returned as it is)
         """
-        return sample

+        return sample

+

 
 
-
[docs]class OnLabelledCollectionProtocol:
+
+

+[docs]
+class OnLabelledCollectionProtocol(AbstractStochasticSeededProtocol):
     """
     Protocols that generate samples from a :class:`qp.data.LabelledCollection` object.
     """
 
     RETURN_TYPES = ['sample_prev', 'labelled_collection', 'index']
 
-
[docs]    def get_labelled_collection(self):
+

+[docs]
+    def get_labelled_collection(self):
         """
         Returns the labelled collection on which this protocol acts.
 
@@ -227,7 +594,10 @@
         """
         return self.data

 
-
[docs]    def on_preclassified_instances(self, pre_classifications, in_place=False):
+
+

+[docs]
+    def on_preclassified_instances(self, pre_classifications, in_place=False):
         """
         Returns a copy of this protocol that acts on a modified version of the original
         :class:`qp.data.LabelledCollection` in which the original instances have been replaced
@@ -250,8 +620,11 @@
             new = deepcopy(self)
             return new.on_preclassified_instances(pre_classifications, in_place=True)

 
-
[docs]    @classmethod
-    def get_collator(cls, return_type='sample_prev'):
+
+

+[docs]
+    @classmethod
+    def get_collator(cls, return_type='sample_prev'):
         """
         Returns a collator function, i.e., a function that prepares the yielded data
 
@@ -264,12 +637,30 @@
         assert return_type in cls.RETURN_TYPES, \
             f'unknown return type passed as argument; valid ones are {cls.RETURN_TYPES}'
         if return_type=='sample_prev':
-            return lambda lc:lc.Xp
+            return lambda lc,params:lc.Xp
         elif return_type=='labelled_collection':
-            return lambda lc:lc

+            return lambda lc,params:lc
+        elif return_type=='index':
+            return lambda lc,params:params

 
 
-
[docs]class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+

+[docs]
+    def sample(self, index):
+        """
+        Realizes the sample given the index of the instances.
+
+        :param index: indexes of the instances to select
+        :return: an instance of :class:`qp.data.LabelledCollection`
+        """
+        return self.data.sampling_from_index(index)

+

+
+
+
+

+[docs]
+class APP(OnLabelledCollectionProtocol):
     """
     Implementation of the artificial prevalence protocol (APP).
     The APP consists of exploring a grid of prevalence values containing `n_prevalences` points (e.g.,
@@ -293,7 +684,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data: LabelledCollection, sample_size=None, n_prevalences=21, repeats=10,
+    def __init__(self, data: LabelledCollection, sample_size=None, n_prevalences=21, repeats=10,
                  smooth_limits_epsilon=0, random_state=0, sanity_check=10000, return_type='sample_prev'):
         super(APP, self).__init__(random_state)
         self.data = data
@@ -313,7 +704,9 @@
 
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def prevalence_grid(self):
+

+[docs]
+    def prevalence_grid(self):
         """
         Generates vectors of prevalence values from an exhaustive grid of prevalence values. The
         number of prevalence values explored for each dimension depends on `n_prevalences`, so that, if, for example,
@@ -338,7 +731,10 @@
             prevs = np.repeat(prevs, self.repeats, axis=0)
         return prevs

 
-
[docs]    def samples_parameters(self):
+
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the APP protocol.
 
@@ -350,25 +746,23 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated
 
         :return: int
         """
-        return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats)

+        return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats)

+

 
 
-
[docs]class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+
+

+[docs]
+class NPP(OnLabelledCollectionProtocol):
     """
     A generator of samples that implements the natural prevalence protocol (NPP). The NPP consists of drawing
     samples uniformly at random, therefore approximately preserving the natural prevalence of the collection.
@@ -383,7 +777,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data:LabelledCollection, sample_size=None, repeats=100, random_state=0,
+    def __init__(self, data:LabelledCollection, sample_size=None, repeats=100, random_state=0,
                  return_type='sample_prev'):
         super(NPP, self).__init__(random_state)
         self.data = data
@@ -392,7 +786,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the NPP protocol.
 
@@ -404,25 +800,23 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats")
 
         :return: int
         """
-        return self.repeats

+        return self.repeats

+

 
 
-
[docs]class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol):
+
+

+[docs]
+class UPP(OnLabelledCollectionProtocol):
     """
     A variant of :class:`APP` that, instead of using a grid of equidistant prevalence values,
     relies on the Kraemer algorithm for sampling unit (k-1)-simplex uniformly at random, with
@@ -441,7 +835,7 @@
         to "labelled_collection" to get instead instances of LabelledCollection
     """
 
-    def __init__(self, data: LabelledCollection, sample_size=None, repeats=100, random_state=0,
+    def __init__(self, data: LabelledCollection, sample_size=None, repeats=100, random_state=0,
                  return_type='sample_prev'):
         super(UPP, self).__init__(random_state)
         self.data = data
@@ -450,7 +844,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the UPP protocol.
 
@@ -462,25 +858,90 @@
             indexes.append(index)
         return indexes

 
-
[docs]    def sample(self, index):
-        """
-        Realizes the sample given the index of the instances.
 
-        :param index: indexes of the instances to select
-        :return: an instance of :class:`qp.data.LabelledCollection`
-        """
-        return self.data.sampling_from_index(index)

-
-
[docs]    def total(self):
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats")
 
         :return: int
         """
-        return self.repeats

+        return self.repeats

+

 
 
-
[docs]class DomainMixer(AbstractStochasticSeededProtocol):
+
+

+[docs]
+class DirichletProtocol(OnLabelledCollectionProtocol):
+    """
+    A protocol that establishes a prior Dirichlet distribution for the prevalence of the samples.
+    Note that providing an all-ones vector of Dirichlet parameters is equivalent to invoking the
+    APP protocol (although each protocol will generate a different series of samples given a
+    fixed seed, since the implementation is different).
+
+    :param data: a `LabelledCollection` from which the samples will be drawn
+    :param alpha: an array-like of shape (n_classes,) with the parameters of the Dirichlet distribution
+    :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from
+        qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised.
+    :param repeats: the number of samples to generate. Default is 100.
+    :param random_state: allows replicating samples across runs (default 0, meaning that the sequence of samples
+        will be the same every time the protocol is called)
+    :param return_type: set to "sample_prev" (default) to get the pairs of (sample, prevalence) at each iteration, or
+        to "labelled_collection" to get instead instances of LabelledCollection
+    """
+
+    def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0,
+                 return_type='sample_prev'):
+        n_classes = data.n_classes
+        if isinstance(alpha, str) and alpha == 'uniform':
+            self.alpha = np.ones(n_classes, dtype=float)
+        elif isinstance(alpha, Number):
+            self.alpha = np.full(n_classes, float(alpha), dtype=float)
+        else:
+            self.alpha = np.asarray(alpha, dtype=float)
+            if self.alpha.ndim != 1 or len(self.alpha) != n_classes:
+                raise ValueError(
+                    f'wrong shape for alpha; expected {n_classes} values, found shape {self.alpha.shape}'
+                )
+
+        super(DirichletProtocol, self).__init__(random_state)
+        self.data = data
+        self.sample_size = qp._get_sample_size(sample_size)
+        self.repeats = repeats
+        self.random_state = random_state
+        self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
+
+

+[docs]
+    def samples_parameters(self):
+        """
+        Return all the necessary parameters to replicate the samples.
+
+        :return: a list of indexes that realize the sampling
+        """
+        prevs = np.random.dirichlet(self.alpha, size=self.repeats)
+        indexes = [self.data.sampling_index(self.sample_size, *prevs_i) for prevs_i in prevs]
+        return indexes

+
+
+

+[docs]
+    def total(self):
+        """
+        Returns the number of samples that will be generated (equals to "repeats")
+
+        :return: int
+        """
+        return self.repeats

+

+
+
+
+

+[docs]
+class DomainMixer(AbstractStochasticSeededProtocol):
     """
     Generates mixtures of two domains (A and B) at controlled rates, but preserving the original class prevalence.
 
@@ -489,7 +950,7 @@
     :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from
         qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised.
     :param repeats: int, number of samples to draw for every mixture rate
-    :param prevalence: the prevalence to preserv along the mixtures. If specified, should be an array containing
+    :param prevalence: the prevalence to preserve along the mixtures. If specified, should be an array containing
         one prevalence value (positive float) for each class and summing up to one. If not specified, the prevalence
         will be taken from the domain A (default).
     :param mixture_points: an integer indicating the number of points to take from a linear scale (e.g., 21 will
@@ -499,7 +960,7 @@
         will be the same every time the protocol is called)
     """
 
-    def __init__(
+    def __init__(
             self,
             domainA: LabelledCollection,
             domainB: LabelledCollection,
@@ -531,7 +992,9 @@
         self.random_state = random_state
         self.collator = OnLabelledCollectionProtocol.get_collator(return_type)
 
-
[docs]    def samples_parameters(self):
+

+[docs]
+    def samples_parameters(self):
         """
         Return all the necessary parameters to replicate the samples as according to the this protocol.
 
@@ -548,7 +1011,10 @@
                 indexesB.append(sampleBidx)
         return list(zip(indexesA, indexesB))

 
-
[docs]    def sample(self, indexes):
+
+

+[docs]
+    def sample(self, indexes):
         """
         Realizes the sample given a pair of indexes of the instances from A and B.
 
@@ -560,13 +1026,18 @@
         sampleB = self.B.sampling_from_index(indexesB)
         return sampleA+sampleB

 
-
[docs]    def total(self):
+
+

+[docs]
+    def total(self):
         """
         Returns the number of samples that will be generated (equals to "repeats * mixture_points")
 
         :return: int
         """
-        return self.repeats * len(self.mixture_points)

+        return self.repeats * len(self.mixture_points)

+

+
 
 
 # aliases
@@ -576,31 +1047,75 @@
 UniformPrevalenceProtocol = UPP
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/quapy/util.html b/docs/build/html/_modules/quapy/util.html index 25532bd..aa2204d 100644 --- a/docs/build/html/_modules/quapy/util.html +++ b/docs/build/html/_modules/quapy/util.html @@ -1,93 +1,395 @@ + - - - - - quapy.util — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation - - + + + + + + + quapy.util — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + - -
- + + + + +
+
+ + + + + + +
+ + + +
+ + +
+ + + +
+
+ + +
+ + + +
+ +
+ +
+ +
    +
    • +
    • +
    • +
+
+ +
    +
  • + + + + + + + + + +
    • +
+ +
+ +
+ + +
+
+
+
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+

Source code for quapy.util

-import contextlib
-import itertools
-import multiprocessing
-import os
-import pickle
-import urllib
-from pathlib import Path
-from contextlib import ExitStack
-import quapy as qp
+import contextlib
+import itertools
+import multiprocessing
+import os
+import pickle
+import urllib
+from pathlib import Path
+from contextlib import ExitStack
 
-import numpy as np
-from joblib import Parallel, delayed
-from time import time
-import signal
+import pandas as pd
+
+import quapy as qp
+
+import numpy as np
+from joblib import Parallel, delayed
+from time import time
+import signal
 
 
-def _get_parallel_slices(n_tasks, n_jobs):
+def _get_parallel_slices(n_tasks, n_jobs):
     if n_jobs == -1:
         n_jobs = multiprocessing.cpu_count()
     batch = int(n_tasks / n_jobs)
@@ -95,7 +397,9 @@
     return [slice(job * batch, (job + 1) * batch + (remainder if job == n_jobs - 1 else 0)) for job in range(n_jobs)]
 
 
-
[docs]def map_parallel(func, args, n_jobs):
+

+[docs]
+def map_parallel(func, args, n_jobs):
     """
     Applies func to n_jobs slices of args. E.g., if args is an array of 99 items and n_jobs=2, then
     func is applied in two parallel processes to args[0:50] and to args[50:99]. func is a function
@@ -113,7 +417,10 @@
     return list(itertools.chain.from_iterable(results))

 
 
-
[docs]def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
+
+

+[docs]
+def parallel(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
     """
     A wrapper of multiprocessing:
 
@@ -129,8 +436,9 @@
     :param seed: the numeric seed
     :param asarray: set to True to return a np.ndarray instead of a list
     :param backend: indicates the backend used for handling parallel works
+    :param open_args: if True, then the delayed function is called on *args_i, instead of on args_i
     """
-    def func_dec(environ, seed, *args):
+    def func_dec(environ, seed, *args):
         qp.environ = environ.copy()
         qp.environ['N_JOBS'] = 1
         #set a context with a temporal seed to ensure results are reproducibles in parallel
@@ -147,8 +455,48 @@
     return out

 
 
-
[docs]@contextlib.contextmanager
-def temp_seed(random_state):
+
+

+[docs]
+def parallel_unpack(func, args, n_jobs, seed=None, asarray=True, backend='loky'):
+    """
+    A wrapper of multiprocessing:
+
+    >>> Parallel(n_jobs=n_jobs)(
+    >>>      delayed(func)(*args_i) for args_i in args
+    >>> )
+
+    that takes the `quapy.environ` variable as input silently.
+    Seeds the child processes to ensure reproducibility when n_jobs>1.
+
+    :param func: callable
+    :param args: args of func
+    :param seed: the numeric seed
+    :param asarray: set to True to return a np.ndarray instead of a list
+    :param backend: indicates the backend used for handling parallel works
+    """
+
+    def func_dec(environ, seed, *args):
+        qp.environ = environ.copy()
+        qp.environ['N_JOBS'] = 1
+        # set a context with a temporal seed to ensure results are reproducibles in parallel
+        with ExitStack() as stack:
+            if seed is not None:
+                stack.enter_context(qp.util.temp_seed(seed))
+            return func(*args)
+
+    out = Parallel(n_jobs=n_jobs, backend=backend)(
+        delayed(func_dec)(qp.environ, None if seed is None else seed + i, *args_i) for i, args_i in enumerate(args)
+    )
+    if asarray:
+        out = np.asarray(out)
+    return out

+
+
+

+[docs]
+@contextlib.contextmanager
+def temp_seed(random_state):
     """
     Can be used in a "with" context to set a temporal seed without modifying the outer numpy's current state. E.g.:
 
@@ -169,14 +517,17 @@
             np.random.set_state(state)

 
 
-
[docs]def download_file(url, archive_filename):
+
+

+[docs]
+def download_file(url, archive_filename):
     """
     Downloads a file from a url
 
     :param url: the url
     :param archive_filename: destination filename
     """
-    def progress(blocknum, bs, size):
+    def progress(blocknum, bs, size):
         total_sz_mb = '%.2f MB' % (size / 1e6)
         current_sz_mb = '%.2f MB' % ((blocknum * bs) / 1e6)
         print('\rdownloaded %s / %s' % (current_sz_mb, total_sz_mb), end='')
@@ -185,9 +536,12 @@
     print("")

 
 
-
[docs]def download_file_if_not_exists(url, archive_filename):
+
+

+[docs]
+def download_file_if_not_exists(url, archive_filename):
     """
-    Dowloads a function (using :meth:`download_file`) if the file does not exist.
+    Downloads a file (using :meth:`download_file`) if the file does not exist.
 
     :param url: the url
     :param archive_filename: destination filename
@@ -198,7 +552,10 @@
     download_file(url, archive_filename)

 
 
-
[docs]def create_if_not_exist(path):
+
+

+[docs]
+def create_if_not_exist(path):
     """
     An alias to `os.makedirs(path, exist_ok=True)` that also returns the path. This is useful in cases like, e.g.:
 
@@ -211,7 +568,10 @@
     return path

 
 
-
[docs]def get_quapy_home():
+
+

+[docs]
+def get_quapy_home():
     """
     Gets the home directory of QuaPy, i.e., the directory where QuaPy saves permanent data, such as dowloaded datasets.
     This directory is `~/quapy_data`
@@ -223,7 +583,10 @@
     return home

 
 
-
[docs]def create_parent_dir(path):
+
+

+[docs]
+def create_parent_dir(path):
     """
     Creates the parent dir (if any) of a given path, if not exists. E.g., for `./path/to/file.txt`, the path `./path/to`
     is created.
@@ -235,7 +598,10 @@
         os.makedirs(parentdir, exist_ok=True)

 
 
-
[docs]def save_text_file(path, text):
+
+

+[docs]
+def save_text_file(path, text):
     """
     Saves a text file to disk, given its full path, and creates the parent directory if missing.
 
@@ -243,11 +609,14 @@
     :param text: text to save.
     """
     create_parent_dir(path)
-    with open(text, 'wt') as fout:
+    with open(path, 'wt') as fout:
         fout.write(text)

 
 
-
[docs]def pickled_resource(pickle_path:str, generation_func:callable, *args):
+
+

+[docs]
+def pickled_resource(pickle_path:str, generation_func:callable, *args):
     """
     Allows for fast reuse of resources that are generated only once by calling generation_func(\\*args). The next times
     this function is invoked, it loads the pickled resource. Example:
@@ -266,15 +635,18 @@
         return generation_func(*args)
     else:
         if os.path.exists(pickle_path):
-            return pickle.load(open(pickle_path, 'rb'))
+            with open(pickle_path, 'rb') as fin:
+                instance = pickle.load(fin)
         else:
             instance = generation_func(*args)
             os.makedirs(str(Path(pickle_path).parent), exist_ok=True)
-            pickle.dump(instance, open(pickle_path, 'wb'), pickle.HIGHEST_PROTOCOL)
-            return instance

+            with open(pickle_path, 'wb') as foo:
+                pickle.dump(instance, foo, pickle.HIGHEST_PROTOCOL)
+        return instance

 
 
-def _check_sample_size(sample_size):
+
+def _check_sample_size(sample_size):
     if sample_size is None:
         assert qp.environ['SAMPLE_SIZE'] is not None, \
             'error: sample_size set to None, and cannot be resolved from the environment'
@@ -284,7 +656,34 @@
     return sample_size
 
 
-
[docs]class EarlyStop:
+

+[docs]
+def load_report(path, as_dict=False):
+    def str2prev_arr(strprev):
+        within = strprev.strip('[]').split()
+        float_list = [float(p) for p in within]
+        float_list[-1] = 1. - sum(float_list[:-1])
+        return np.asarray(float_list)
+
+    df = pd.read_csv(path, index_col=0)
+    df['true-prev'] = df['true-prev'].apply(str2prev_arr)
+    df['estim-prev'] = df['estim-prev'].apply(str2prev_arr)
+    if as_dict:
+        d = {}
+        for col in df.columns.values:
+            vals = df[col].values
+            if col in ['true-prev', 'estim-prev']:
+                vals = np.vstack(vals)
+            d[col] = vals
+        return d
+    else:
+        return df

+
+
+
+

+[docs]
+class EarlyStop:
     """
     A class implementing the early-stopping condition typically used for training neural networks.
 
@@ -309,7 +708,7 @@
     :ivar IMPROVED: flag (boolean) indicating whether there was an improvement in the last call
     """
 
-    def __init__(self, patience, lower_is_better=True):
+    def __init__(self, patience, lower_is_better=True):
 
         self.PATIENCE_LIMIT = patience
         self.better = lambda a,b: a<b if lower_is_better else a>b
@@ -319,7 +718,7 @@
         self.STOP = False
         self.IMPROVED = False
 
-    def __call__(self, watch_score, epoch):
+    def __call__(self, watch_score, epoch):
         """
         Commits the new score found in epoch `epoch`. If the score improves over the best score found so far, then
         the patiente counter gets reset. If otherwise, the patience counter is decreased, and in case it reachs 0,
@@ -339,8 +738,11 @@
                 self.STOP = True

 
 
-
[docs]@contextlib.contextmanager
-def timeout(seconds):
+
+

+[docs]
+@contextlib.contextmanager
+def timeout(seconds):
     """
     Opens a context that will launch an exception if not closed after a given number of seconds
 
@@ -359,7 +761,7 @@
     :param seconds: number of seconds, set to <=0 to ignore the timer
     """
     if seconds > 0:
-        def handler(signum, frame):
+        def handler(signum, frame):
             raise TimeoutError()
 
         signal.signal(signal.SIGALRM, handler)
@@ -370,33 +772,78 @@
     if seconds > 0:
         signal.alarm(0)

 
+
-
+ + + + + + +
+ +
+
+
+ +
+ + + +
-
- -
- -
-

© Copyright 2024, Alejandro Moreo.

+
+ +
+ + +
+ + + + - Built with Sphinx using a - theme - provided by Read the Docs. - + -
- - - - +

+ + © Copyright 2024, Alejandro Moreo. +
+ +

+ + +
- +

+ Created using Sphinx 9.0.4. +
+

+
+ + + + + +
+ +
+

+ + Built with the PyData Sphinx Theme 0.20.0. +

+ +
+ + + + + \ No newline at end of file diff --git a/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js b/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js index 8549469..8141580 100644 --- a/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js +++ b/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js @@ -1,20 +1,9 @@ -/* - * _sphinx_javascript_frameworks_compat.js - * ~~~~~~~~~~ - * - * Compatability shim for jQuery and underscores.js. - * - * WILL BE REMOVED IN Sphinx 6.0 - * xref RemovedInSphinx60Warning +/* Compatability shim for jQuery and underscores.js. * + * Copyright Sphinx contributors + * Released under the two clause BSD licence */ -/** - * select a different prefix for underscore - */ -$u = _.noConflict(); - - /** * small helper function to urldecode strings * diff --git a/docs/build/html/_static/css/badge_only.css b/docs/build/html/_static/css/badge_only.css index c718cee..88ba55b 100644 --- a/docs/build/html/_static/css/badge_only.css +++ b/docs/build/html/_static/css/badge_only.css @@ -1 +1 @@ -.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} \ No newline at end of file +.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px} \ No newline at end of file diff --git a/docs/build/html/_static/css/theme.css b/docs/build/html/_static/css/theme.css index 19a446a..a88467c 100644 --- a/docs/build/html/_static/css/theme.css +++ b/docs/build/html/_static/css/theme.css @@ -1,4 +1,4 @@ html{box-sizing:border-box}*,:after,:before{box-sizing:inherit}article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}[hidden],audio:not([controls]){display:none}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}blockquote{margin:0}dfn{font-style:italic}ins{background:#ff9;text-decoration:none}ins,mark{color:#000}mark{background:#ff0;font-style:italic;font-weight:700}.rst-content code,.rst-content tt,code,kbd,pre,samp{font-family:monospace,serif;_font-family:courier new,monospace;font-size:1em}pre{white-space:pre}q{quotes:none}q:after,q:before{content:"";content:none}small{font-size:85%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}dl,ol,ul{margin:0;padding:0;list-style:none;list-style-image:none}li{list-style:none}dd{margin:0}img{border:0;-ms-interpolation-mode:bicubic;vertical-align:middle;max-width:100%}svg:not(:root){overflow:hidden}figure,form{margin:0}label{cursor:pointer}button,input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}button,input{line-height:normal}button,input[type=button],input[type=reset],input[type=submit]{cursor:pointer;-webkit-appearance:button;*overflow:visible}button[disabled],input[disabled]{cursor:default}input[type=search]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}textarea{resize:vertical}table{border-collapse:collapse;border-spacing:0}td{vertical-align:top}.chromeframe{margin:.2em 0;background:#ccc;color:#000;padding:.2em 0}.ir{display:block;border:0;text-indent:-999em;overflow:hidden;background-color:transparent;background-repeat:no-repeat;text-align:left;direction:ltr;*line-height:0}.ir br{display:none}.hidden{display:none!important;visibility:hidden}.visuallyhidden{border:0;clip:rect(0 0 0 0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.visuallyhidden.focusable:active,.visuallyhidden.focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.invisible{visibility:hidden}.relative{position:relative}big,small{font-size:100%}@media print{body,html,section{background:none!important}*{box-shadow:none!important;text-shadow:none!important;filter:none!important;-ms-filter:none!important}a,a:visited{text-decoration:underline}.ir a:after,a[href^="#"]:after,a[href^="javascript:"]:after{content:""}blockquote,pre{page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}img{max-width:100%!important}@page{margin:.5cm}.rst-content .toctree-wrapper>p.caption,h2,h3,p{orphans:3;widows:3}.rst-content .toctree-wrapper>p.caption,h2,h3{page-break-after:avoid}}.btn,.fa:before,.icon:before,.rst-content .admonition,.rst-content .admonition-title:before,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .code-block-caption .headerlink:before,.rst-content .danger,.rst-content .eqno .headerlink:before,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-alert,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before,input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week],select,textarea{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}/*! * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) - */@font-face{font-family:FontAwesome;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713);src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix&v=4.7.0) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#fontawesomeregular) format("svg");font-weight:400;font-style:normal}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14286em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14286em;width:2.14286em;top:.14286em;text-align:center}.fa-li.fa-lg{left:-1.85714em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa-pull-left.icon,.fa.fa-pull-left,.rst-content .code-block-caption .fa-pull-left.headerlink,.rst-content .eqno .fa-pull-left.headerlink,.rst-content .fa-pull-left.admonition-title,.rst-content code.download span.fa-pull-left:first-child,.rst-content dl dt .fa-pull-left.headerlink,.rst-content h1 .fa-pull-left.headerlink,.rst-content h2 .fa-pull-left.headerlink,.rst-content h3 .fa-pull-left.headerlink,.rst-content h4 .fa-pull-left.headerlink,.rst-content h5 .fa-pull-left.headerlink,.rst-content h6 .fa-pull-left.headerlink,.rst-content p .fa-pull-left.headerlink,.rst-content table>caption .fa-pull-left.headerlink,.rst-content tt.download span.fa-pull-left:first-child,.wy-menu-vertical li.current>a button.fa-pull-left.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-left.toctree-expand,.wy-menu-vertical li button.fa-pull-left.toctree-expand{margin-right:.3em}.fa-pull-right.icon,.fa.fa-pull-right,.rst-content .code-block-caption .fa-pull-right.headerlink,.rst-content .eqno .fa-pull-right.headerlink,.rst-content .fa-pull-right.admonition-title,.rst-content code.download span.fa-pull-right:first-child,.rst-content dl dt .fa-pull-right.headerlink,.rst-content h1 .fa-pull-right.headerlink,.rst-content h2 .fa-pull-right.headerlink,.rst-content h3 .fa-pull-right.headerlink,.rst-content h4 .fa-pull-right.headerlink,.rst-content h5 .fa-pull-right.headerlink,.rst-content h6 .fa-pull-right.headerlink,.rst-content p .fa-pull-right.headerlink,.rst-content table>caption .fa-pull-right.headerlink,.rst-content tt.download span.fa-pull-right:first-child,.wy-menu-vertical li.current>a button.fa-pull-right.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-right.toctree-expand,.wy-menu-vertical li button.fa-pull-right.toctree-expand{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left,.pull-left.icon,.rst-content .code-block-caption .pull-left.headerlink,.rst-content .eqno .pull-left.headerlink,.rst-content .pull-left.admonition-title,.rst-content code.download span.pull-left:first-child,.rst-content dl dt .pull-left.headerlink,.rst-content h1 .pull-left.headerlink,.rst-content h2 .pull-left.headerlink,.rst-content h3 .pull-left.headerlink,.rst-content h4 .pull-left.headerlink,.rst-content h5 .pull-left.headerlink,.rst-content h6 .pull-left.headerlink,.rst-content p .pull-left.headerlink,.rst-content table>caption .pull-left.headerlink,.rst-content tt.download span.pull-left:first-child,.wy-menu-vertical li.current>a button.pull-left.toctree-expand,.wy-menu-vertical li.on a button.pull-left.toctree-expand,.wy-menu-vertical li button.pull-left.toctree-expand{margin-right:.3em}.fa.pull-right,.pull-right.icon,.rst-content .code-block-caption .pull-right.headerlink,.rst-content .eqno .pull-right.headerlink,.rst-content .pull-right.admonition-title,.rst-content code.download span.pull-right:first-child,.rst-content dl dt .pull-right.headerlink,.rst-content h1 .pull-right.headerlink,.rst-content h2 .pull-right.headerlink,.rst-content h3 .pull-right.headerlink,.rst-content h4 .pull-right.headerlink,.rst-content h5 .pull-right.headerlink,.rst-content h6 .pull-right.headerlink,.rst-content p .pull-right.headerlink,.rst-content table>caption .pull-right.headerlink,.rst-content tt.download span.pull-right:first-child,.wy-menu-vertical li.current>a button.pull-right.toctree-expand,.wy-menu-vertical li.on a button.pull-right.toctree-expand,.wy-menu-vertical li button.pull-right.toctree-expand{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s linear infinite;animation:fa-spin 2s linear infinite}.fa-pulse{-webkit-animation:fa-spin 1s steps(8) infinite;animation:fa-spin 1s steps(8) infinite}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scaleX(-1);-ms-transform:scaleX(-1);transform:scaleX(-1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scaleY(-1);-ms-transform:scaleY(-1);transform:scaleY(-1)}:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:""}.fa-music:before{content:""}.fa-search:before,.icon-search:before{content:""}.fa-envelope-o:before{content:""}.fa-heart:before{content:""}.fa-star:before{content:""}.fa-star-o:before{content:""}.fa-user:before{content:""}.fa-film:before{content:""}.fa-th-large:before{content:""}.fa-th:before{content:""}.fa-th-list:before{content:""}.fa-check:before{content:""}.fa-close:before,.fa-remove:before,.fa-times:before{content:""}.fa-search-plus:before{content:""}.fa-search-minus:before{content:""}.fa-power-off:before{content:""}.fa-signal:before{content:""}.fa-cog:before,.fa-gear:before{content:""}.fa-trash-o:before{content:""}.fa-home:before,.icon-home:before{content:""}.fa-file-o:before{content:""}.fa-clock-o:before{content:""}.fa-road:before{content:""}.fa-download:before,.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{content:""}.fa-arrow-circle-o-down:before{content:""}.fa-arrow-circle-o-up:before{content:""}.fa-inbox:before{content:""}.fa-play-circle-o:before{content:""}.fa-repeat:before,.fa-rotate-right:before{content:""}.fa-refresh:before{content:""}.fa-list-alt:before{content:""}.fa-lock:before{content:""}.fa-flag:before{content:""}.fa-headphones:before{content:""}.fa-volume-off:before{content:""}.fa-volume-down:before{content:""}.fa-volume-up:before{content:""}.fa-qrcode:before{content:""}.fa-barcode:before{content:""}.fa-tag:before{content:""}.fa-tags:before{content:""}.fa-book:before,.icon-book:before{content:""}.fa-bookmark:before{content:""}.fa-print:before{content:""}.fa-camera:before{content:""}.fa-font:before{content:""}.fa-bold:before{content:""}.fa-italic:before{content:""}.fa-text-height:before{content:""}.fa-text-width:before{content:""}.fa-align-left:before{content:""}.fa-align-center:before{content:""}.fa-align-right:before{content:""}.fa-align-justify:before{content:""}.fa-list:before{content:""}.fa-dedent:before,.fa-outdent:before{content:""}.fa-indent:before{content:""}.fa-video-camera:before{content:""}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:""}.fa-pencil:before{content:""}.fa-map-marker:before{content:""}.fa-adjust:before{content:""}.fa-tint:before{content:""}.fa-edit:before,.fa-pencil-square-o:before{content:""}.fa-share-square-o:before{content:""}.fa-check-square-o:before{content:""}.fa-arrows:before{content:""}.fa-step-backward:before{content:""}.fa-fast-backward:before{content:""}.fa-backward:before{content:""}.fa-play:before{content:""}.fa-pause:before{content:""}.fa-stop:before{content:""}.fa-forward:before{content:""}.fa-fast-forward:before{content:""}.fa-step-forward:before{content:""}.fa-eject:before{content:""}.fa-chevron-left:before{content:""}.fa-chevron-right:before{content:""}.fa-plus-circle:before{content:""}.fa-minus-circle:before{content:""}.fa-times-circle:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before{content:""}.fa-check-circle:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before{content:""}.fa-question-circle:before{content:""}.fa-info-circle:before{content:""}.fa-crosshairs:before{content:""}.fa-times-circle-o:before{content:""}.fa-check-circle-o:before{content:""}.fa-ban:before{content:""}.fa-arrow-left:before{content:""}.fa-arrow-right:before{content:""}.fa-arrow-up:before{content:""}.fa-arrow-down:before{content:""}.fa-mail-forward:before,.fa-share:before{content:""}.fa-expand:before{content:""}.fa-compress:before{content:""}.fa-plus:before{content:""}.fa-minus:before{content:""}.fa-asterisk:before{content:""}.fa-exclamation-circle:before,.rst-content .admonition-title:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before{content:""}.fa-gift:before{content:""}.fa-leaf:before{content:""}.fa-fire:before,.icon-fire:before{content:""}.fa-eye:before{content:""}.fa-eye-slash:before{content:""}.fa-exclamation-triangle:before,.fa-warning:before{content:""}.fa-plane:before{content:""}.fa-calendar:before{content:""}.fa-random:before{content:""}.fa-comment:before{content:""}.fa-magnet:before{content:""}.fa-chevron-up:before{content:""}.fa-chevron-down:before{content:""}.fa-retweet:before{content:""}.fa-shopping-cart:before{content:""}.fa-folder:before{content:""}.fa-folder-open:before{content:""}.fa-arrows-v:before{content:""}.fa-arrows-h:before{content:""}.fa-bar-chart-o:before,.fa-bar-chart:before{content:""}.fa-twitter-square:before{content:""}.fa-facebook-square:before{content:""}.fa-camera-retro:before{content:""}.fa-key:before{content:""}.fa-cogs:before,.fa-gears:before{content:""}.fa-comments:before{content:""}.fa-thumbs-o-up:before{content:""}.fa-thumbs-o-down:before{content:""}.fa-star-half:before{content:""}.fa-heart-o:before{content:""}.fa-sign-out:before{content:""}.fa-linkedin-square:before{content:""}.fa-thumb-tack:before{content:""}.fa-external-link:before{content:""}.fa-sign-in:before{content:""}.fa-trophy:before{content:""}.fa-github-square:before{content:""}.fa-upload:before{content:""}.fa-lemon-o:before{content:""}.fa-phone:before{content:""}.fa-square-o:before{content:""}.fa-bookmark-o:before{content:""}.fa-phone-square:before{content:""}.fa-twitter:before{content:""}.fa-facebook-f:before,.fa-facebook:before{content:""}.fa-github:before,.icon-github:before{content:""}.fa-unlock:before{content:""}.fa-credit-card:before{content:""}.fa-feed:before,.fa-rss:before{content:""}.fa-hdd-o:before{content:""}.fa-bullhorn:before{content:""}.fa-bell:before{content:""}.fa-certificate:before{content:""}.fa-hand-o-right:before{content:""}.fa-hand-o-left:before{content:""}.fa-hand-o-up:before{content:""}.fa-hand-o-down:before{content:""}.fa-arrow-circle-left:before,.icon-circle-arrow-left:before{content:""}.fa-arrow-circle-right:before,.icon-circle-arrow-right:before{content:""}.fa-arrow-circle-up:before{content:""}.fa-arrow-circle-down:before{content:""}.fa-globe:before{content:""}.fa-wrench:before{content:""}.fa-tasks:before{content:""}.fa-filter:before{content:""}.fa-briefcase:before{content:""}.fa-arrows-alt:before{content:""}.fa-group:before,.fa-users:before{content:""}.fa-chain:before,.fa-link:before,.icon-link:before{content:""}.fa-cloud:before{content:""}.fa-flask:before{content:""}.fa-cut:before,.fa-scissors:before{content:""}.fa-copy:before,.fa-files-o:before{content:""}.fa-paperclip:before{content:""}.fa-floppy-o:before,.fa-save:before{content:""}.fa-square:before{content:""}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:""}.fa-list-ul:before{content:""}.fa-list-ol:before{content:""}.fa-strikethrough:before{content:""}.fa-underline:before{content:""}.fa-table:before{content:""}.fa-magic:before{content:""}.fa-truck:before{content:""}.fa-pinterest:before{content:""}.fa-pinterest-square:before{content:""}.fa-google-plus-square:before{content:""}.fa-google-plus:before{content:""}.fa-money:before{content:""}.fa-caret-down:before,.icon-caret-down:before,.wy-dropdown .caret:before{content:""}.fa-caret-up:before{content:""}.fa-caret-left:before{content:""}.fa-caret-right:before{content:""}.fa-columns:before{content:""}.fa-sort:before,.fa-unsorted:before{content:""}.fa-sort-desc:before,.fa-sort-down:before{content:""}.fa-sort-asc:before,.fa-sort-up:before{content:""}.fa-envelope:before{content:""}.fa-linkedin:before{content:""}.fa-rotate-left:before,.fa-undo:before{content:""}.fa-gavel:before,.fa-legal:before{content:""}.fa-dashboard:before,.fa-tachometer:before{content:""}.fa-comment-o:before{content:""}.fa-comments-o:before{content:""}.fa-bolt:before,.fa-flash:before{content:""}.fa-sitemap:before{content:""}.fa-umbrella:before{content:""}.fa-clipboard:before,.fa-paste:before{content:""}.fa-lightbulb-o:before{content:""}.fa-exchange:before{content:""}.fa-cloud-download:before{content:""}.fa-cloud-upload:before{content:""}.fa-user-md:before{content:""}.fa-stethoscope:before{content:""}.fa-suitcase:before{content:""}.fa-bell-o:before{content:""}.fa-coffee:before{content:""}.fa-cutlery:before{content:""}.fa-file-text-o:before{content:""}.fa-building-o:before{content:""}.fa-hospital-o:before{content:""}.fa-ambulance:before{content:""}.fa-medkit:before{content:""}.fa-fighter-jet:before{content:""}.fa-beer:before{content:""}.fa-h-square:before{content:""}.fa-plus-square:before{content:""}.fa-angle-double-left:before{content:""}.fa-angle-double-right:before{content:""}.fa-angle-double-up:before{content:""}.fa-angle-double-down:before{content:""}.fa-angle-left:before{content:""}.fa-angle-right:before{content:""}.fa-angle-up:before{content:""}.fa-angle-down:before{content:""}.fa-desktop:before{content:""}.fa-laptop:before{content:""}.fa-tablet:before{content:""}.fa-mobile-phone:before,.fa-mobile:before{content:""}.fa-circle-o:before{content:""}.fa-quote-left:before{content:""}.fa-quote-right:before{content:""}.fa-spinner:before{content:""}.fa-circle:before{content:""}.fa-mail-reply:before,.fa-reply:before{content:""}.fa-github-alt:before{content:""}.fa-folder-o:before{content:""}.fa-folder-open-o:before{content:""}.fa-smile-o:before{content:""}.fa-frown-o:before{content:""}.fa-meh-o:before{content:""}.fa-gamepad:before{content:""}.fa-keyboard-o:before{content:""}.fa-flag-o:before{content:""}.fa-flag-checkered:before{content:""}.fa-terminal:before{content:""}.fa-code:before{content:""}.fa-mail-reply-all:before,.fa-reply-all:before{content:""}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:""}.fa-location-arrow:before{content:""}.fa-crop:before{content:""}.fa-code-fork:before{content:""}.fa-chain-broken:before,.fa-unlink:before{content:""}.fa-question:before{content:""}.fa-info:before{content:""}.fa-exclamation:before{content:""}.fa-superscript:before{content:""}.fa-subscript:before{content:""}.fa-eraser:before{content:""}.fa-puzzle-piece:before{content:""}.fa-microphone:before{content:""}.fa-microphone-slash:before{content:""}.fa-shield:before{content:""}.fa-calendar-o:before{content:""}.fa-fire-extinguisher:before{content:""}.fa-rocket:before{content:""}.fa-maxcdn:before{content:""}.fa-chevron-circle-left:before{content:""}.fa-chevron-circle-right:before{content:""}.fa-chevron-circle-up:before{content:""}.fa-chevron-circle-down:before{content:""}.fa-html5:before{content:""}.fa-css3:before{content:""}.fa-anchor:before{content:""}.fa-unlock-alt:before{content:""}.fa-bullseye:before{content:""}.fa-ellipsis-h:before{content:""}.fa-ellipsis-v:before{content:""}.fa-rss-square:before{content:""}.fa-play-circle:before{content:""}.fa-ticket:before{content:""}.fa-minus-square:before{content:""}.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before{content:""}.fa-level-up:before{content:""}.fa-level-down:before{content:""}.fa-check-square:before{content:""}.fa-pencil-square:before{content:""}.fa-external-link-square:before{content:""}.fa-share-square:before{content:""}.fa-compass:before{content:""}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:""}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:""}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:""}.fa-eur:before,.fa-euro:before{content:""}.fa-gbp:before{content:""}.fa-dollar:before,.fa-usd:before{content:""}.fa-inr:before,.fa-rupee:before{content:""}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:""}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:""}.fa-krw:before,.fa-won:before{content:""}.fa-bitcoin:before,.fa-btc:before{content:""}.fa-file:before{content:""}.fa-file-text:before{content:""}.fa-sort-alpha-asc:before{content:""}.fa-sort-alpha-desc:before{content:""}.fa-sort-amount-asc:before{content:""}.fa-sort-amount-desc:before{content:""}.fa-sort-numeric-asc:before{content:""}.fa-sort-numeric-desc:before{content:""}.fa-thumbs-up:before{content:""}.fa-thumbs-down:before{content:""}.fa-youtube-square:before{content:""}.fa-youtube:before{content:""}.fa-xing:before{content:""}.fa-xing-square:before{content:""}.fa-youtube-play:before{content:""}.fa-dropbox:before{content:""}.fa-stack-overflow:before{content:""}.fa-instagram:before{content:""}.fa-flickr:before{content:""}.fa-adn:before{content:""}.fa-bitbucket:before,.icon-bitbucket:before{content:""}.fa-bitbucket-square:before{content:""}.fa-tumblr:before{content:""}.fa-tumblr-square:before{content:""}.fa-long-arrow-down:before{content:""}.fa-long-arrow-up:before{content:""}.fa-long-arrow-left:before{content:""}.fa-long-arrow-right:before{content:""}.fa-apple:before{content:""}.fa-windows:before{content:""}.fa-android:before{content:""}.fa-linux:before{content:""}.fa-dribbble:before{content:""}.fa-skype:before{content:""}.fa-foursquare:before{content:""}.fa-trello:before{content:""}.fa-female:before{content:""}.fa-male:before{content:""}.fa-gittip:before,.fa-gratipay:before{content:""}.fa-sun-o:before{content:""}.fa-moon-o:before{content:""}.fa-archive:before{content:""}.fa-bug:before{content:""}.fa-vk:before{content:""}.fa-weibo:before{content:""}.fa-renren:before{content:""}.fa-pagelines:before{content:""}.fa-stack-exchange:before{content:""}.fa-arrow-circle-o-right:before{content:""}.fa-arrow-circle-o-left:before{content:""}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:""}.fa-dot-circle-o:before{content:""}.fa-wheelchair:before{content:""}.fa-vimeo-square:before{content:""}.fa-try:before,.fa-turkish-lira:before{content:""}.fa-plus-square-o:before,.wy-menu-vertical li button.toctree-expand:before{content:""}.fa-space-shuttle:before{content:""}.fa-slack:before{content:""}.fa-envelope-square:before{content:""}.fa-wordpress:before{content:""}.fa-openid:before{content:""}.fa-bank:before,.fa-institution:before,.fa-university:before{content:""}.fa-graduation-cap:before,.fa-mortar-board:before{content:""}.fa-yahoo:before{content:""}.fa-google:before{content:""}.fa-reddit:before{content:""}.fa-reddit-square:before{content:""}.fa-stumbleupon-circle:before{content:""}.fa-stumbleupon:before{content:""}.fa-delicious:before{content:""}.fa-digg:before{content:""}.fa-pied-piper-pp:before{content:""}.fa-pied-piper-alt:before{content:""}.fa-drupal:before{content:""}.fa-joomla:before{content:""}.fa-language:before{content:""}.fa-fax:before{content:""}.fa-building:before{content:""}.fa-child:before{content:""}.fa-paw:before{content:""}.fa-spoon:before{content:""}.fa-cube:before{content:""}.fa-cubes:before{content:""}.fa-behance:before{content:""}.fa-behance-square:before{content:""}.fa-steam:before{content:""}.fa-steam-square:before{content:""}.fa-recycle:before{content:""}.fa-automobile:before,.fa-car:before{content:""}.fa-cab:before,.fa-taxi:before{content:""}.fa-tree:before{content:""}.fa-spotify:before{content:""}.fa-deviantart:before{content:""}.fa-soundcloud:before{content:""}.fa-database:before{content:""}.fa-file-pdf-o:before{content:""}.fa-file-word-o:before{content:""}.fa-file-excel-o:before{content:""}.fa-file-powerpoint-o:before{content:""}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:""}.fa-file-archive-o:before,.fa-file-zip-o:before{content:""}.fa-file-audio-o:before,.fa-file-sound-o:before{content:""}.fa-file-movie-o:before,.fa-file-video-o:before{content:""}.fa-file-code-o:before{content:""}.fa-vine:before{content:""}.fa-codepen:before{content:""}.fa-jsfiddle:before{content:""}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:""}.fa-circle-o-notch:before{content:""}.fa-ra:before,.fa-rebel:before,.fa-resistance:before{content:""}.fa-empire:before,.fa-ge:before{content:""}.fa-git-square:before{content:""}.fa-git:before{content:""}.fa-hacker-news:before,.fa-y-combinator-square:before,.fa-yc-square:before{content:""}.fa-tencent-weibo:before{content:""}.fa-qq:before{content:""}.fa-wechat:before,.fa-weixin:before{content:""}.fa-paper-plane:before,.fa-send:before{content:""}.fa-paper-plane-o:before,.fa-send-o:before{content:""}.fa-history:before{content:""}.fa-circle-thin:before{content:""}.fa-header:before{content:""}.fa-paragraph:before{content:""}.fa-sliders:before{content:""}.fa-share-alt:before{content:""}.fa-share-alt-square:before{content:""}.fa-bomb:before{content:""}.fa-futbol-o:before,.fa-soccer-ball-o:before{content:""}.fa-tty:before{content:""}.fa-binoculars:before{content:""}.fa-plug:before{content:""}.fa-slideshare:before{content:""}.fa-twitch:before{content:""}.fa-yelp:before{content:""}.fa-newspaper-o:before{content:""}.fa-wifi:before{content:""}.fa-calculator:before{content:""}.fa-paypal:before{content:""}.fa-google-wallet:before{content:""}.fa-cc-visa:before{content:""}.fa-cc-mastercard:before{content:""}.fa-cc-discover:before{content:""}.fa-cc-amex:before{content:""}.fa-cc-paypal:before{content:""}.fa-cc-stripe:before{content:""}.fa-bell-slash:before{content:""}.fa-bell-slash-o:before{content:""}.fa-trash:before{content:""}.fa-copyright:before{content:""}.fa-at:before{content:""}.fa-eyedropper:before{content:""}.fa-paint-brush:before{content:""}.fa-birthday-cake:before{content:""}.fa-area-chart:before{content:""}.fa-pie-chart:before{content:""}.fa-line-chart:before{content:""}.fa-lastfm:before{content:""}.fa-lastfm-square:before{content:""}.fa-toggle-off:before{content:""}.fa-toggle-on:before{content:""}.fa-bicycle:before{content:""}.fa-bus:before{content:""}.fa-ioxhost:before{content:""}.fa-angellist:before{content:""}.fa-cc:before{content:""}.fa-ils:before,.fa-shekel:before,.fa-sheqel:before{content:""}.fa-meanpath:before{content:""}.fa-buysellads:before{content:""}.fa-connectdevelop:before{content:""}.fa-dashcube:before{content:""}.fa-forumbee:before{content:""}.fa-leanpub:before{content:""}.fa-sellsy:before{content:""}.fa-shirtsinbulk:before{content:""}.fa-simplybuilt:before{content:""}.fa-skyatlas:before{content:""}.fa-cart-plus:before{content:""}.fa-cart-arrow-down:before{content:""}.fa-diamond:before{content:""}.fa-ship:before{content:""}.fa-user-secret:before{content:""}.fa-motorcycle:before{content:""}.fa-street-view:before{content:""}.fa-heartbeat:before{content:""}.fa-venus:before{content:""}.fa-mars:before{content:""}.fa-mercury:before{content:""}.fa-intersex:before,.fa-transgender:before{content:""}.fa-transgender-alt:before{content:""}.fa-venus-double:before{content:""}.fa-mars-double:before{content:""}.fa-venus-mars:before{content:""}.fa-mars-stroke:before{content:""}.fa-mars-stroke-v:before{content:""}.fa-mars-stroke-h:before{content:""}.fa-neuter:before{content:""}.fa-genderless:before{content:""}.fa-facebook-official:before{content:""}.fa-pinterest-p:before{content:""}.fa-whatsapp:before{content:""}.fa-server:before{content:""}.fa-user-plus:before{content:""}.fa-user-times:before{content:""}.fa-bed:before,.fa-hotel:before{content:""}.fa-viacoin:before{content:""}.fa-train:before{content:""}.fa-subway:before{content:""}.fa-medium:before{content:""}.fa-y-combinator:before,.fa-yc:before{content:""}.fa-optin-monster:before{content:""}.fa-opencart:before{content:""}.fa-expeditedssl:before{content:""}.fa-battery-4:before,.fa-battery-full:before,.fa-battery:before{content:""}.fa-battery-3:before,.fa-battery-three-quarters:before{content:""}.fa-battery-2:before,.fa-battery-half:before{content:""}.fa-battery-1:before,.fa-battery-quarter:before{content:""}.fa-battery-0:before,.fa-battery-empty:before{content:""}.fa-mouse-pointer:before{content:""}.fa-i-cursor:before{content:""}.fa-object-group:before{content:""}.fa-object-ungroup:before{content:""}.fa-sticky-note:before{content:""}.fa-sticky-note-o:before{content:""}.fa-cc-jcb:before{content:""}.fa-cc-diners-club:before{content:""}.fa-clone:before{content:""}.fa-balance-scale:before{content:""}.fa-hourglass-o:before{content:""}.fa-hourglass-1:before,.fa-hourglass-start:before{content:""}.fa-hourglass-2:before,.fa-hourglass-half:before{content:""}.fa-hourglass-3:before,.fa-hourglass-end:before{content:""}.fa-hourglass:before{content:""}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:""}.fa-hand-paper-o:before,.fa-hand-stop-o:before{content:""}.fa-hand-scissors-o:before{content:""}.fa-hand-lizard-o:before{content:""}.fa-hand-spock-o:before{content:""}.fa-hand-pointer-o:before{content:""}.fa-hand-peace-o:before{content:""}.fa-trademark:before{content:""}.fa-registered:before{content:""}.fa-creative-commons:before{content:""}.fa-gg:before{content:""}.fa-gg-circle:before{content:""}.fa-tripadvisor:before{content:""}.fa-odnoklassniki:before{content:""}.fa-odnoklassniki-square:before{content:""}.fa-get-pocket:before{content:""}.fa-wikipedia-w:before{content:""}.fa-safari:before{content:""}.fa-chrome:before{content:""}.fa-firefox:before{content:""}.fa-opera:before{content:""}.fa-internet-explorer:before{content:""}.fa-television:before,.fa-tv:before{content:""}.fa-contao:before{content:""}.fa-500px:before{content:""}.fa-amazon:before{content:""}.fa-calendar-plus-o:before{content:""}.fa-calendar-minus-o:before{content:""}.fa-calendar-times-o:before{content:""}.fa-calendar-check-o:before{content:""}.fa-industry:before{content:""}.fa-map-pin:before{content:""}.fa-map-signs:before{content:""}.fa-map-o:before{content:""}.fa-map:before{content:""}.fa-commenting:before{content:""}.fa-commenting-o:before{content:""}.fa-houzz:before{content:""}.fa-vimeo:before{content:""}.fa-black-tie:before{content:""}.fa-fonticons:before{content:""}.fa-reddit-alien:before{content:""}.fa-edge:before{content:""}.fa-credit-card-alt:before{content:""}.fa-codiepie:before{content:""}.fa-modx:before{content:""}.fa-fort-awesome:before{content:""}.fa-usb:before{content:""}.fa-product-hunt:before{content:""}.fa-mixcloud:before{content:""}.fa-scribd:before{content:""}.fa-pause-circle:before{content:""}.fa-pause-circle-o:before{content:""}.fa-stop-circle:before{content:""}.fa-stop-circle-o:before{content:""}.fa-shopping-bag:before{content:""}.fa-shopping-basket:before{content:""}.fa-hashtag:before{content:""}.fa-bluetooth:before{content:""}.fa-bluetooth-b:before{content:""}.fa-percent:before{content:""}.fa-gitlab:before,.icon-gitlab:before{content:""}.fa-wpbeginner:before{content:""}.fa-wpforms:before{content:""}.fa-envira:before{content:""}.fa-universal-access:before{content:""}.fa-wheelchair-alt:before{content:""}.fa-question-circle-o:before{content:""}.fa-blind:before{content:""}.fa-audio-description:before{content:""}.fa-volume-control-phone:before{content:""}.fa-braille:before{content:""}.fa-assistive-listening-systems:before{content:""}.fa-american-sign-language-interpreting:before,.fa-asl-interpreting:before{content:""}.fa-deaf:before,.fa-deafness:before,.fa-hard-of-hearing:before{content:""}.fa-glide:before{content:""}.fa-glide-g:before{content:""}.fa-sign-language:before,.fa-signing:before{content:""}.fa-low-vision:before{content:""}.fa-viadeo:before{content:""}.fa-viadeo-square:before{content:""}.fa-snapchat:before{content:""}.fa-snapchat-ghost:before{content:""}.fa-snapchat-square:before{content:""}.fa-pied-piper:before{content:""}.fa-first-order:before{content:""}.fa-yoast:before{content:""}.fa-themeisle:before{content:""}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:""}.fa-fa:before,.fa-font-awesome:before{content:""}.fa-handshake-o:before{content:""}.fa-envelope-open:before{content:""}.fa-envelope-open-o:before{content:""}.fa-linode:before{content:""}.fa-address-book:before{content:""}.fa-address-book-o:before{content:""}.fa-address-card:before,.fa-vcard:before{content:""}.fa-address-card-o:before,.fa-vcard-o:before{content:""}.fa-user-circle:before{content:""}.fa-user-circle-o:before{content:""}.fa-user-o:before{content:""}.fa-id-badge:before{content:""}.fa-drivers-license:before,.fa-id-card:before{content:""}.fa-drivers-license-o:before,.fa-id-card-o:before{content:""}.fa-quora:before{content:""}.fa-free-code-camp:before{content:""}.fa-telegram:before{content:""}.fa-thermometer-4:before,.fa-thermometer-full:before,.fa-thermometer:before{content:""}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:""}.fa-thermometer-2:before,.fa-thermometer-half:before{content:""}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:""}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:""}.fa-shower:before{content:""}.fa-bath:before,.fa-bathtub:before,.fa-s15:before{content:""}.fa-podcast:before{content:""}.fa-window-maximize:before{content:""}.fa-window-minimize:before{content:""}.fa-window-restore:before{content:""}.fa-times-rectangle:before,.fa-window-close:before{content:""}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:""}.fa-bandcamp:before{content:""}.fa-grav:before{content:""}.fa-etsy:before{content:""}.fa-imdb:before{content:""}.fa-ravelry:before{content:""}.fa-eercast:before{content:""}.fa-microchip:before{content:""}.fa-snowflake-o:before{content:""}.fa-superpowers:before{content:""}.fa-wpexplorer:before{content:""}.fa-meetup:before{content:""}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-dropdown .caret,.wy-inline-validate.wy-inline-validate-danger .wy-input-context,.wy-inline-validate.wy-inline-validate-info .wy-input-context,.wy-inline-validate.wy-inline-validate-success .wy-input-context,.wy-inline-validate.wy-inline-validate-warning .wy-input-context,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{font-family:inherit}.fa:before,.icon:before,.rst-content .admonition-title:before,.rst-content .code-block-caption .headerlink:before,.rst-content .eqno .headerlink:before,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before{font-family:FontAwesome;display:inline-block;font-style:normal;font-weight:400;line-height:1;text-decoration:inherit}.rst-content .code-block-caption a .headerlink,.rst-content .eqno a .headerlink,.rst-content a .admonition-title,.rst-content code.download a span:first-child,.rst-content dl dt a .headerlink,.rst-content h1 a .headerlink,.rst-content h2 a .headerlink,.rst-content h3 a .headerlink,.rst-content h4 a .headerlink,.rst-content h5 a .headerlink,.rst-content h6 a .headerlink,.rst-content p.caption a .headerlink,.rst-content p a .headerlink,.rst-content table>caption a .headerlink,.rst-content tt.download a span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li a button.toctree-expand,a .fa,a .icon,a .rst-content .admonition-title,a .rst-content .code-block-caption .headerlink,a .rst-content .eqno .headerlink,a .rst-content code.download span:first-child,a .rst-content dl dt .headerlink,a .rst-content h1 .headerlink,a .rst-content h2 .headerlink,a .rst-content h3 .headerlink,a .rst-content h4 .headerlink,a .rst-content h5 .headerlink,a .rst-content h6 .headerlink,a .rst-content p.caption .headerlink,a .rst-content p .headerlink,a .rst-content table>caption .headerlink,a .rst-content tt.download span:first-child,a .wy-menu-vertical li button.toctree-expand{display:inline-block;text-decoration:inherit}.btn .fa,.btn .icon,.btn .rst-content .admonition-title,.btn .rst-content .code-block-caption .headerlink,.btn .rst-content .eqno .headerlink,.btn .rst-content code.download span:first-child,.btn .rst-content dl dt .headerlink,.btn .rst-content h1 .headerlink,.btn .rst-content h2 .headerlink,.btn .rst-content h3 .headerlink,.btn .rst-content h4 .headerlink,.btn .rst-content h5 .headerlink,.btn .rst-content h6 .headerlink,.btn .rst-content p .headerlink,.btn .rst-content table>caption .headerlink,.btn .rst-content tt.download span:first-child,.btn .wy-menu-vertical li.current>a button.toctree-expand,.btn .wy-menu-vertical li.on a button.toctree-expand,.btn .wy-menu-vertical li button.toctree-expand,.nav .fa,.nav .icon,.nav .rst-content .admonition-title,.nav .rst-content .code-block-caption .headerlink,.nav .rst-content .eqno .headerlink,.nav .rst-content code.download span:first-child,.nav .rst-content dl dt .headerlink,.nav .rst-content h1 .headerlink,.nav .rst-content h2 .headerlink,.nav .rst-content h3 .headerlink,.nav .rst-content h4 .headerlink,.nav .rst-content h5 .headerlink,.nav .rst-content h6 .headerlink,.nav .rst-content p .headerlink,.nav .rst-content table>caption .headerlink,.nav .rst-content tt.download span:first-child,.nav .wy-menu-vertical li.current>a button.toctree-expand,.nav .wy-menu-vertical li.on a button.toctree-expand,.nav .wy-menu-vertical li button.toctree-expand,.rst-content .btn .admonition-title,.rst-content .code-block-caption .btn .headerlink,.rst-content .code-block-caption .nav .headerlink,.rst-content .eqno .btn .headerlink,.rst-content .eqno .nav .headerlink,.rst-content .nav .admonition-title,.rst-content code.download .btn span:first-child,.rst-content code.download .nav span:first-child,.rst-content dl dt .btn .headerlink,.rst-content dl dt .nav .headerlink,.rst-content h1 .btn .headerlink,.rst-content h1 .nav .headerlink,.rst-content h2 .btn .headerlink,.rst-content h2 .nav .headerlink,.rst-content h3 .btn .headerlink,.rst-content h3 .nav .headerlink,.rst-content h4 .btn .headerlink,.rst-content h4 .nav .headerlink,.rst-content h5 .btn .headerlink,.rst-content h5 .nav .headerlink,.rst-content h6 .btn .headerlink,.rst-content h6 .nav .headerlink,.rst-content p .btn .headerlink,.rst-content p .nav .headerlink,.rst-content table>caption .btn .headerlink,.rst-content table>caption .nav .headerlink,.rst-content tt.download .btn span:first-child,.rst-content tt.download .nav span:first-child,.wy-menu-vertical li .btn button.toctree-expand,.wy-menu-vertical li.current>a .btn button.toctree-expand,.wy-menu-vertical li.current>a .nav button.toctree-expand,.wy-menu-vertical li .nav button.toctree-expand,.wy-menu-vertical li.on a .btn button.toctree-expand,.wy-menu-vertical li.on a .nav button.toctree-expand{display:inline}.btn .fa-large.icon,.btn .fa.fa-large,.btn .rst-content .code-block-caption .fa-large.headerlink,.btn .rst-content .eqno .fa-large.headerlink,.btn .rst-content .fa-large.admonition-title,.btn .rst-content code.download span.fa-large:first-child,.btn .rst-content dl dt .fa-large.headerlink,.btn .rst-content h1 .fa-large.headerlink,.btn .rst-content h2 .fa-large.headerlink,.btn .rst-content h3 .fa-large.headerlink,.btn .rst-content h4 .fa-large.headerlink,.btn .rst-content h5 .fa-large.headerlink,.btn .rst-content h6 .fa-large.headerlink,.btn .rst-content p .fa-large.headerlink,.btn .rst-content table>caption .fa-large.headerlink,.btn .rst-content tt.download span.fa-large:first-child,.btn .wy-menu-vertical li button.fa-large.toctree-expand,.nav .fa-large.icon,.nav .fa.fa-large,.nav .rst-content .code-block-caption .fa-large.headerlink,.nav .rst-content .eqno .fa-large.headerlink,.nav .rst-content .fa-large.admonition-title,.nav .rst-content code.download span.fa-large:first-child,.nav .rst-content dl dt .fa-large.headerlink,.nav .rst-content h1 .fa-large.headerlink,.nav .rst-content h2 .fa-large.headerlink,.nav .rst-content h3 .fa-large.headerlink,.nav .rst-content h4 .fa-large.headerlink,.nav .rst-content h5 .fa-large.headerlink,.nav .rst-content h6 .fa-large.headerlink,.nav .rst-content p .fa-large.headerlink,.nav .rst-content table>caption .fa-large.headerlink,.nav .rst-content tt.download span.fa-large:first-child,.nav .wy-menu-vertical li button.fa-large.toctree-expand,.rst-content .btn .fa-large.admonition-title,.rst-content .code-block-caption .btn .fa-large.headerlink,.rst-content .code-block-caption .nav .fa-large.headerlink,.rst-content .eqno .btn .fa-large.headerlink,.rst-content .eqno .nav .fa-large.headerlink,.rst-content .nav .fa-large.admonition-title,.rst-content code.download .btn span.fa-large:first-child,.rst-content code.download .nav span.fa-large:first-child,.rst-content dl dt .btn .fa-large.headerlink,.rst-content dl dt .nav .fa-large.headerlink,.rst-content h1 .btn .fa-large.headerlink,.rst-content h1 .nav .fa-large.headerlink,.rst-content h2 .btn .fa-large.headerlink,.rst-content h2 .nav .fa-large.headerlink,.rst-content h3 .btn .fa-large.headerlink,.rst-content h3 .nav .fa-large.headerlink,.rst-content h4 .btn .fa-large.headerlink,.rst-content h4 .nav .fa-large.headerlink,.rst-content h5 .btn .fa-large.headerlink,.rst-content h5 .nav .fa-large.headerlink,.rst-content h6 .btn .fa-large.headerlink,.rst-content h6 .nav .fa-large.headerlink,.rst-content p .btn .fa-large.headerlink,.rst-content p .nav .fa-large.headerlink,.rst-content table>caption .btn .fa-large.headerlink,.rst-content table>caption .nav .fa-large.headerlink,.rst-content tt.download .btn span.fa-large:first-child,.rst-content tt.download .nav span.fa-large:first-child,.wy-menu-vertical li .btn button.fa-large.toctree-expand,.wy-menu-vertical li .nav button.fa-large.toctree-expand{line-height:.9em}.btn .fa-spin.icon,.btn .fa.fa-spin,.btn .rst-content .code-block-caption .fa-spin.headerlink,.btn .rst-content .eqno .fa-spin.headerlink,.btn .rst-content .fa-spin.admonition-title,.btn .rst-content code.download span.fa-spin:first-child,.btn .rst-content dl dt .fa-spin.headerlink,.btn .rst-content h1 .fa-spin.headerlink,.btn .rst-content h2 .fa-spin.headerlink,.btn .rst-content h3 .fa-spin.headerlink,.btn .rst-content h4 .fa-spin.headerlink,.btn .rst-content h5 .fa-spin.headerlink,.btn .rst-content h6 .fa-spin.headerlink,.btn .rst-content p .fa-spin.headerlink,.btn .rst-content table>caption .fa-spin.headerlink,.btn .rst-content tt.download span.fa-spin:first-child,.btn .wy-menu-vertical li button.fa-spin.toctree-expand,.nav .fa-spin.icon,.nav .fa.fa-spin,.nav .rst-content .code-block-caption .fa-spin.headerlink,.nav .rst-content .eqno .fa-spin.headerlink,.nav .rst-content .fa-spin.admonition-title,.nav .rst-content code.download span.fa-spin:first-child,.nav .rst-content dl dt .fa-spin.headerlink,.nav .rst-content h1 .fa-spin.headerlink,.nav .rst-content h2 .fa-spin.headerlink,.nav .rst-content h3 .fa-spin.headerlink,.nav .rst-content h4 .fa-spin.headerlink,.nav .rst-content h5 .fa-spin.headerlink,.nav .rst-content h6 .fa-spin.headerlink,.nav .rst-content p .fa-spin.headerlink,.nav .rst-content table>caption .fa-spin.headerlink,.nav .rst-content tt.download span.fa-spin:first-child,.nav .wy-menu-vertical li button.fa-spin.toctree-expand,.rst-content .btn .fa-spin.admonition-title,.rst-content .code-block-caption .btn .fa-spin.headerlink,.rst-content .code-block-caption .nav .fa-spin.headerlink,.rst-content .eqno .btn .fa-spin.headerlink,.rst-content .eqno .nav .fa-spin.headerlink,.rst-content .nav .fa-spin.admonition-title,.rst-content code.download .btn span.fa-spin:first-child,.rst-content code.download .nav span.fa-spin:first-child,.rst-content dl dt .btn .fa-spin.headerlink,.rst-content dl dt .nav .fa-spin.headerlink,.rst-content h1 .btn .fa-spin.headerlink,.rst-content h1 .nav .fa-spin.headerlink,.rst-content h2 .btn .fa-spin.headerlink,.rst-content h2 .nav .fa-spin.headerlink,.rst-content h3 .btn .fa-spin.headerlink,.rst-content h3 .nav .fa-spin.headerlink,.rst-content h4 .btn .fa-spin.headerlink,.rst-content h4 .nav .fa-spin.headerlink,.rst-content h5 .btn .fa-spin.headerlink,.rst-content h5 .nav .fa-spin.headerlink,.rst-content h6 .btn .fa-spin.headerlink,.rst-content h6 .nav .fa-spin.headerlink,.rst-content p .btn .fa-spin.headerlink,.rst-content p .nav .fa-spin.headerlink,.rst-content table>caption .btn .fa-spin.headerlink,.rst-content table>caption .nav .fa-spin.headerlink,.rst-content tt.download .btn span.fa-spin:first-child,.rst-content tt.download .nav span.fa-spin:first-child,.wy-menu-vertical li .btn button.fa-spin.toctree-expand,.wy-menu-vertical li .nav button.fa-spin.toctree-expand{display:inline-block}.btn.fa:before,.btn.icon:before,.rst-content .btn.admonition-title:before,.rst-content .code-block-caption .btn.headerlink:before,.rst-content .eqno .btn.headerlink:before,.rst-content code.download span.btn:first-child:before,.rst-content dl dt .btn.headerlink:before,.rst-content h1 .btn.headerlink:before,.rst-content h2 .btn.headerlink:before,.rst-content h3 .btn.headerlink:before,.rst-content h4 .btn.headerlink:before,.rst-content h5 .btn.headerlink:before,.rst-content h6 .btn.headerlink:before,.rst-content p .btn.headerlink:before,.rst-content table>caption .btn.headerlink:before,.rst-content tt.download span.btn:first-child:before,.wy-menu-vertical li button.btn.toctree-expand:before{opacity:.5;-webkit-transition:opacity .05s ease-in;-moz-transition:opacity .05s ease-in;transition:opacity .05s ease-in}.btn.fa:hover:before,.btn.icon:hover:before,.rst-content .btn.admonition-title:hover:before,.rst-content .code-block-caption .btn.headerlink:hover:before,.rst-content .eqno .btn.headerlink:hover:before,.rst-content code.download span.btn:first-child:hover:before,.rst-content dl dt .btn.headerlink:hover:before,.rst-content h1 .btn.headerlink:hover:before,.rst-content h2 .btn.headerlink:hover:before,.rst-content h3 .btn.headerlink:hover:before,.rst-content h4 .btn.headerlink:hover:before,.rst-content h5 .btn.headerlink:hover:before,.rst-content h6 .btn.headerlink:hover:before,.rst-content p .btn.headerlink:hover:before,.rst-content table>caption .btn.headerlink:hover:before,.rst-content tt.download span.btn:first-child:hover:before,.wy-menu-vertical li button.btn.toctree-expand:hover:before{opacity:1}.btn-mini .fa:before,.btn-mini .icon:before,.btn-mini .rst-content .admonition-title:before,.btn-mini .rst-content .code-block-caption .headerlink:before,.btn-mini .rst-content .eqno .headerlink:before,.btn-mini .rst-content code.download span:first-child:before,.btn-mini .rst-content dl dt .headerlink:before,.btn-mini .rst-content h1 .headerlink:before,.btn-mini .rst-content h2 .headerlink:before,.btn-mini .rst-content h3 .headerlink:before,.btn-mini .rst-content h4 .headerlink:before,.btn-mini .rst-content h5 .headerlink:before,.btn-mini .rst-content h6 .headerlink:before,.btn-mini .rst-content p .headerlink:before,.btn-mini .rst-content table>caption .headerlink:before,.btn-mini .rst-content tt.download span:first-child:before,.btn-mini .wy-menu-vertical li button.toctree-expand:before,.rst-content .btn-mini .admonition-title:before,.rst-content .code-block-caption .btn-mini .headerlink:before,.rst-content .eqno .btn-mini .headerlink:before,.rst-content code.download .btn-mini span:first-child:before,.rst-content dl dt .btn-mini .headerlink:before,.rst-content h1 .btn-mini .headerlink:before,.rst-content h2 .btn-mini .headerlink:before,.rst-content h3 .btn-mini .headerlink:before,.rst-content h4 .btn-mini .headerlink:before,.rst-content h5 .btn-mini .headerlink:before,.rst-content h6 .btn-mini .headerlink:before,.rst-content p .btn-mini .headerlink:before,.rst-content table>caption .btn-mini .headerlink:before,.rst-content tt.download .btn-mini span:first-child:before,.wy-menu-vertical li .btn-mini button.toctree-expand:before{font-size:14px;vertical-align:-15%}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.wy-alert{padding:12px;line-height:24px;margin-bottom:24px;background:#e7f2fa}.rst-content .admonition-title,.wy-alert-title{font-weight:700;display:block;color:#fff;background:#6ab0de;padding:6px 12px;margin:-12px -12px 12px}.rst-content .danger,.rst-content .error,.rst-content .wy-alert-danger.admonition,.rst-content .wy-alert-danger.admonition-todo,.rst-content .wy-alert-danger.attention,.rst-content .wy-alert-danger.caution,.rst-content .wy-alert-danger.hint,.rst-content .wy-alert-danger.important,.rst-content .wy-alert-danger.note,.rst-content .wy-alert-danger.seealso,.rst-content .wy-alert-danger.tip,.rst-content .wy-alert-danger.warning,.wy-alert.wy-alert-danger{background:#fdf3f2}.rst-content .danger .admonition-title,.rst-content .danger .wy-alert-title,.rst-content .error .admonition-title,.rst-content .error .wy-alert-title,.rst-content .wy-alert-danger.admonition-todo .admonition-title,.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,.rst-content .wy-alert-danger.admonition .admonition-title,.rst-content .wy-alert-danger.admonition .wy-alert-title,.rst-content .wy-alert-danger.attention .admonition-title,.rst-content .wy-alert-danger.attention .wy-alert-title,.rst-content .wy-alert-danger.caution .admonition-title,.rst-content .wy-alert-danger.caution .wy-alert-title,.rst-content .wy-alert-danger.hint .admonition-title,.rst-content .wy-alert-danger.hint .wy-alert-title,.rst-content .wy-alert-danger.important .admonition-title,.rst-content .wy-alert-danger.important .wy-alert-title,.rst-content .wy-alert-danger.note .admonition-title,.rst-content .wy-alert-danger.note .wy-alert-title,.rst-content .wy-alert-danger.seealso .admonition-title,.rst-content .wy-alert-danger.seealso .wy-alert-title,.rst-content .wy-alert-danger.tip .admonition-title,.rst-content .wy-alert-danger.tip .wy-alert-title,.rst-content .wy-alert-danger.warning .admonition-title,.rst-content .wy-alert-danger.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-danger .admonition-title,.wy-alert.wy-alert-danger .rst-content .admonition-title,.wy-alert.wy-alert-danger .wy-alert-title{background:#f29f97}.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .warning,.rst-content .wy-alert-warning.admonition,.rst-content .wy-alert-warning.danger,.rst-content .wy-alert-warning.error,.rst-content .wy-alert-warning.hint,.rst-content .wy-alert-warning.important,.rst-content .wy-alert-warning.note,.rst-content .wy-alert-warning.seealso,.rst-content .wy-alert-warning.tip,.wy-alert.wy-alert-warning{background:#ffedcc}.rst-content .admonition-todo .admonition-title,.rst-content .admonition-todo .wy-alert-title,.rst-content .attention .admonition-title,.rst-content .attention .wy-alert-title,.rst-content .caution .admonition-title,.rst-content .caution .wy-alert-title,.rst-content .warning .admonition-title,.rst-content .warning .wy-alert-title,.rst-content .wy-alert-warning.admonition .admonition-title,.rst-content .wy-alert-warning.admonition .wy-alert-title,.rst-content .wy-alert-warning.danger .admonition-title,.rst-content .wy-alert-warning.danger .wy-alert-title,.rst-content .wy-alert-warning.error .admonition-title,.rst-content .wy-alert-warning.error .wy-alert-title,.rst-content .wy-alert-warning.hint .admonition-title,.rst-content .wy-alert-warning.hint .wy-alert-title,.rst-content .wy-alert-warning.important .admonition-title,.rst-content .wy-alert-warning.important .wy-alert-title,.rst-content .wy-alert-warning.note .admonition-title,.rst-content .wy-alert-warning.note .wy-alert-title,.rst-content .wy-alert-warning.seealso .admonition-title,.rst-content .wy-alert-warning.seealso .wy-alert-title,.rst-content .wy-alert-warning.tip .admonition-title,.rst-content .wy-alert-warning.tip .wy-alert-title,.rst-content .wy-alert.wy-alert-warning .admonition-title,.wy-alert.wy-alert-warning .rst-content .admonition-title,.wy-alert.wy-alert-warning .wy-alert-title{background:#f0b37e}.rst-content .note,.rst-content .seealso,.rst-content .wy-alert-info.admonition,.rst-content .wy-alert-info.admonition-todo,.rst-content .wy-alert-info.attention,.rst-content .wy-alert-info.caution,.rst-content .wy-alert-info.danger,.rst-content .wy-alert-info.error,.rst-content .wy-alert-info.hint,.rst-content .wy-alert-info.important,.rst-content .wy-alert-info.tip,.rst-content .wy-alert-info.warning,.wy-alert.wy-alert-info{background:#e7f2fa}.rst-content .note .admonition-title,.rst-content .note .wy-alert-title,.rst-content .seealso .admonition-title,.rst-content .seealso .wy-alert-title,.rst-content .wy-alert-info.admonition-todo .admonition-title,.rst-content .wy-alert-info.admonition-todo .wy-alert-title,.rst-content .wy-alert-info.admonition .admonition-title,.rst-content .wy-alert-info.admonition .wy-alert-title,.rst-content .wy-alert-info.attention .admonition-title,.rst-content .wy-alert-info.attention .wy-alert-title,.rst-content .wy-alert-info.caution .admonition-title,.rst-content .wy-alert-info.caution .wy-alert-title,.rst-content .wy-alert-info.danger .admonition-title,.rst-content .wy-alert-info.danger .wy-alert-title,.rst-content .wy-alert-info.error .admonition-title,.rst-content .wy-alert-info.error .wy-alert-title,.rst-content .wy-alert-info.hint .admonition-title,.rst-content .wy-alert-info.hint .wy-alert-title,.rst-content .wy-alert-info.important .admonition-title,.rst-content .wy-alert-info.important .wy-alert-title,.rst-content .wy-alert-info.tip .admonition-title,.rst-content .wy-alert-info.tip .wy-alert-title,.rst-content .wy-alert-info.warning .admonition-title,.rst-content .wy-alert-info.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-info .admonition-title,.wy-alert.wy-alert-info .rst-content .admonition-title,.wy-alert.wy-alert-info .wy-alert-title{background:#6ab0de}.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .wy-alert-success.admonition,.rst-content .wy-alert-success.admonition-todo,.rst-content .wy-alert-success.attention,.rst-content .wy-alert-success.caution,.rst-content .wy-alert-success.danger,.rst-content .wy-alert-success.error,.rst-content .wy-alert-success.note,.rst-content .wy-alert-success.seealso,.rst-content .wy-alert-success.warning,.wy-alert.wy-alert-success{background:#dbfaf4}.rst-content .hint .admonition-title,.rst-content .hint .wy-alert-title,.rst-content .important .admonition-title,.rst-content .important .wy-alert-title,.rst-content .tip .admonition-title,.rst-content .tip .wy-alert-title,.rst-content .wy-alert-success.admonition-todo .admonition-title,.rst-content .wy-alert-success.admonition-todo .wy-alert-title,.rst-content .wy-alert-success.admonition .admonition-title,.rst-content .wy-alert-success.admonition .wy-alert-title,.rst-content .wy-alert-success.attention .admonition-title,.rst-content .wy-alert-success.attention .wy-alert-title,.rst-content .wy-alert-success.caution .admonition-title,.rst-content .wy-alert-success.caution .wy-alert-title,.rst-content .wy-alert-success.danger .admonition-title,.rst-content .wy-alert-success.danger .wy-alert-title,.rst-content .wy-alert-success.error .admonition-title,.rst-content .wy-alert-success.error .wy-alert-title,.rst-content .wy-alert-success.note .admonition-title,.rst-content .wy-alert-success.note .wy-alert-title,.rst-content .wy-alert-success.seealso .admonition-title,.rst-content .wy-alert-success.seealso .wy-alert-title,.rst-content .wy-alert-success.warning .admonition-title,.rst-content .wy-alert-success.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-success .admonition-title,.wy-alert.wy-alert-success .rst-content .admonition-title,.wy-alert.wy-alert-success .wy-alert-title{background:#1abc9c}.rst-content .wy-alert-neutral.admonition,.rst-content .wy-alert-neutral.admonition-todo,.rst-content .wy-alert-neutral.attention,.rst-content .wy-alert-neutral.caution,.rst-content .wy-alert-neutral.danger,.rst-content .wy-alert-neutral.error,.rst-content .wy-alert-neutral.hint,.rst-content .wy-alert-neutral.important,.rst-content .wy-alert-neutral.note,.rst-content .wy-alert-neutral.seealso,.rst-content .wy-alert-neutral.tip,.rst-content .wy-alert-neutral.warning,.wy-alert.wy-alert-neutral{background:#f3f6f6}.rst-content .wy-alert-neutral.admonition-todo .admonition-title,.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,.rst-content .wy-alert-neutral.admonition .admonition-title,.rst-content .wy-alert-neutral.admonition .wy-alert-title,.rst-content .wy-alert-neutral.attention .admonition-title,.rst-content .wy-alert-neutral.attention .wy-alert-title,.rst-content .wy-alert-neutral.caution .admonition-title,.rst-content .wy-alert-neutral.caution .wy-alert-title,.rst-content .wy-alert-neutral.danger .admonition-title,.rst-content .wy-alert-neutral.danger .wy-alert-title,.rst-content .wy-alert-neutral.error .admonition-title,.rst-content .wy-alert-neutral.error .wy-alert-title,.rst-content .wy-alert-neutral.hint .admonition-title,.rst-content .wy-alert-neutral.hint .wy-alert-title,.rst-content .wy-alert-neutral.important .admonition-title,.rst-content .wy-alert-neutral.important .wy-alert-title,.rst-content .wy-alert-neutral.note .admonition-title,.rst-content .wy-alert-neutral.note .wy-alert-title,.rst-content .wy-alert-neutral.seealso .admonition-title,.rst-content .wy-alert-neutral.seealso .wy-alert-title,.rst-content .wy-alert-neutral.tip .admonition-title,.rst-content .wy-alert-neutral.tip .wy-alert-title,.rst-content .wy-alert-neutral.warning .admonition-title,.rst-content .wy-alert-neutral.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-neutral .admonition-title,.wy-alert.wy-alert-neutral .rst-content .admonition-title,.wy-alert.wy-alert-neutral .wy-alert-title{color:#404040;background:#e1e4e5}.rst-content .wy-alert-neutral.admonition-todo a,.rst-content .wy-alert-neutral.admonition a,.rst-content .wy-alert-neutral.attention a,.rst-content .wy-alert-neutral.caution a,.rst-content .wy-alert-neutral.danger a,.rst-content .wy-alert-neutral.error a,.rst-content .wy-alert-neutral.hint a,.rst-content .wy-alert-neutral.important a,.rst-content .wy-alert-neutral.note a,.rst-content .wy-alert-neutral.seealso a,.rst-content .wy-alert-neutral.tip a,.rst-content .wy-alert-neutral.warning a,.wy-alert.wy-alert-neutral a{color:#2980b9}.rst-content .admonition-todo p:last-child,.rst-content .admonition p:last-child,.rst-content .attention p:last-child,.rst-content .caution p:last-child,.rst-content .danger p:last-child,.rst-content .error p:last-child,.rst-content .hint p:last-child,.rst-content .important p:last-child,.rst-content .note p:last-child,.rst-content .seealso p:last-child,.rst-content .tip p:last-child,.rst-content .warning p:last-child,.wy-alert p:last-child{margin-bottom:0}.wy-tray-container{position:fixed;bottom:0;left:0;z-index:600}.wy-tray-container li{display:block;width:300px;background:transparent;color:#fff;text-align:center;box-shadow:0 5px 5px 0 rgba(0,0,0,.1);padding:0 24px;min-width:20%;opacity:0;height:0;line-height:56px;overflow:hidden;-webkit-transition:all .3s ease-in;-moz-transition:all .3s ease-in;transition:all .3s ease-in}.wy-tray-container li.wy-tray-item-success{background:#27ae60}.wy-tray-container li.wy-tray-item-info{background:#2980b9}.wy-tray-container li.wy-tray-item-warning{background:#e67e22}.wy-tray-container li.wy-tray-item-danger{background:#e74c3c}.wy-tray-container li.on{opacity:1;height:56px}@media screen and (max-width:768px){.wy-tray-container{bottom:auto;top:0;width:100%}.wy-tray-container li{width:100%}}button{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle;cursor:pointer;line-height:normal;-webkit-appearance:button;*overflow:visible}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}button[disabled]{cursor:default}.btn{display:inline-block;border-radius:2px;line-height:normal;white-space:nowrap;text-align:center;cursor:pointer;font-size:100%;padding:6px 12px 8px;color:#fff;border:1px solid rgba(0,0,0,.1);background-color:#27ae60;text-decoration:none;font-weight:400;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 2px -1px hsla(0,0%,100%,.5),inset 0 -2px 0 0 rgba(0,0,0,.1);outline-none:false;vertical-align:middle;*display:inline;zoom:1;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:all .1s linear;-moz-transition:all .1s linear;transition:all .1s linear}.btn-hover{background:#2e8ece;color:#fff}.btn:hover{background:#2cc36b;color:#fff}.btn:focus{background:#2cc36b;outline:0}.btn:active{box-shadow:inset 0 -1px 0 0 rgba(0,0,0,.05),inset 0 2px 0 0 rgba(0,0,0,.1);padding:8px 12px 6px}.btn:visited{color:#fff}.btn-disabled,.btn-disabled:active,.btn-disabled:focus,.btn-disabled:hover,.btn:disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:.4;cursor:not-allowed;box-shadow:none}.btn::-moz-focus-inner{padding:0;border:0}.btn-small{font-size:80%}.btn-info{background-color:#2980b9!important}.btn-info:hover{background-color:#2e8ece!important}.btn-neutral{background-color:#f3f6f6!important;color:#404040!important}.btn-neutral:hover{background-color:#e5ebeb!important;color:#404040}.btn-neutral:visited{color:#404040!important}.btn-success{background-color:#27ae60!important}.btn-success:hover{background-color:#295!important}.btn-danger{background-color:#e74c3c!important}.btn-danger:hover{background-color:#ea6153!important}.btn-warning{background-color:#e67e22!important}.btn-warning:hover{background-color:#e98b39!important}.btn-invert{background-color:#222}.btn-invert:hover{background-color:#2f2f2f!important}.btn-link{background-color:transparent!important;color:#2980b9;box-shadow:none;border-color:transparent!important}.btn-link:active,.btn-link:hover{background-color:transparent!important;color:#409ad5!important;box-shadow:none}.btn-link:visited{color:#9b59b6}.wy-btn-group .btn,.wy-control .btn{vertical-align:middle}.wy-btn-group{margin-bottom:24px;*zoom:1}.wy-btn-group:after,.wy-btn-group:before{display:table;content:""}.wy-btn-group:after{clear:both}.wy-dropdown{position:relative;display:inline-block}.wy-dropdown-active .wy-dropdown-menu{display:block}.wy-dropdown-menu{position:absolute;left:0;display:none;float:left;top:100%;min-width:100%;background:#fcfcfc;z-index:100;border:1px solid #cfd7dd;box-shadow:0 2px 2px 0 rgba(0,0,0,.1);padding:12px}.wy-dropdown-menu>dd>a{display:block;clear:both;color:#404040;white-space:nowrap;font-size:90%;padding:0 12px;cursor:pointer}.wy-dropdown-menu>dd>a:hover{background:#2980b9;color:#fff}.wy-dropdown-menu>dd.divider{border-top:1px solid #cfd7dd;margin:6px 0}.wy-dropdown-menu>dd.search{padding-bottom:12px}.wy-dropdown-menu>dd.search input[type=search]{width:100%}.wy-dropdown-menu>dd.call-to-action{background:#e3e3e3;text-transform:uppercase;font-weight:500;font-size:80%}.wy-dropdown-menu>dd.call-to-action:hover{background:#e3e3e3}.wy-dropdown-menu>dd.call-to-action .btn{color:#fff}.wy-dropdown.wy-dropdown-up .wy-dropdown-menu{bottom:100%;top:auto;left:auto;right:0}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu{background:#fcfcfc;margin-top:2px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a{padding:6px 12px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover{background:#2980b9;color:#fff}.wy-dropdown.wy-dropdown-left .wy-dropdown-menu{right:0;left:auto;text-align:right}.wy-dropdown-arrow:before{content:" ";border-bottom:5px solid #f5f5f5;border-left:5px solid transparent;border-right:5px solid transparent;position:absolute;display:block;top:-4px;left:50%;margin-left:-3px}.wy-dropdown-arrow.wy-dropdown-arrow-left:before{left:11px}.wy-form-stacked select{display:block}.wy-form-aligned .wy-help-inline,.wy-form-aligned input,.wy-form-aligned label,.wy-form-aligned select,.wy-form-aligned textarea{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-form-aligned .wy-control-group>label{display:inline-block;vertical-align:middle;width:10em;margin:6px 12px 0 0;float:left}.wy-form-aligned .wy-control{float:left}.wy-form-aligned .wy-control label{display:block}.wy-form-aligned .wy-control select{margin-top:6px}fieldset{margin:0}fieldset,legend{border:0;padding:0}legend{width:100%;white-space:normal;margin-bottom:24px;font-size:150%;*margin-left:-7px}label,legend{display:block}label{margin:0 0 .3125em;color:#333;font-size:90%}input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}.wy-control-group{margin-bottom:24px;max-width:1200px;margin-left:auto;margin-right:auto;*zoom:1}.wy-control-group:after,.wy-control-group:before{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group.wy-control-group-required>label:after{content:" *";color:#e74c3c}.wy-control-group .wy-form-full,.wy-control-group .wy-form-halves,.wy-control-group .wy-form-thirds{padding-bottom:12px}.wy-control-group .wy-form-full input[type=color],.wy-control-group .wy-form-full input[type=date],.wy-control-group .wy-form-full input[type=datetime-local],.wy-control-group .wy-form-full input[type=datetime],.wy-control-group .wy-form-full input[type=email],.wy-control-group .wy-form-full input[type=month],.wy-control-group .wy-form-full input[type=number],.wy-control-group .wy-form-full input[type=password],.wy-control-group .wy-form-full input[type=search],.wy-control-group .wy-form-full input[type=tel],.wy-control-group .wy-form-full input[type=text],.wy-control-group .wy-form-full input[type=time],.wy-control-group .wy-form-full input[type=url],.wy-control-group .wy-form-full input[type=week],.wy-control-group .wy-form-full select,.wy-control-group .wy-form-halves input[type=color],.wy-control-group .wy-form-halves input[type=date],.wy-control-group .wy-form-halves input[type=datetime-local],.wy-control-group .wy-form-halves input[type=datetime],.wy-control-group .wy-form-halves input[type=email],.wy-control-group .wy-form-halves input[type=month],.wy-control-group .wy-form-halves input[type=number],.wy-control-group .wy-form-halves input[type=password],.wy-control-group .wy-form-halves input[type=search],.wy-control-group .wy-form-halves input[type=tel],.wy-control-group .wy-form-halves input[type=text],.wy-control-group .wy-form-halves input[type=time],.wy-control-group .wy-form-halves input[type=url],.wy-control-group .wy-form-halves input[type=week],.wy-control-group .wy-form-halves select,.wy-control-group .wy-form-thirds input[type=color],.wy-control-group .wy-form-thirds input[type=date],.wy-control-group .wy-form-thirds input[type=datetime-local],.wy-control-group .wy-form-thirds input[type=datetime],.wy-control-group .wy-form-thirds input[type=email],.wy-control-group .wy-form-thirds input[type=month],.wy-control-group .wy-form-thirds input[type=number],.wy-control-group .wy-form-thirds input[type=password],.wy-control-group .wy-form-thirds input[type=search],.wy-control-group .wy-form-thirds input[type=tel],.wy-control-group .wy-form-thirds input[type=text],.wy-control-group .wy-form-thirds input[type=time],.wy-control-group .wy-form-thirds input[type=url],.wy-control-group .wy-form-thirds input[type=week],.wy-control-group .wy-form-thirds select{width:100%}.wy-control-group .wy-form-full{float:left;display:block;width:100%;margin-right:0}.wy-control-group .wy-form-full:last-child{margin-right:0}.wy-control-group .wy-form-halves{float:left;display:block;margin-right:2.35765%;width:48.82117%}.wy-control-group .wy-form-halves:last-child,.wy-control-group .wy-form-halves:nth-of-type(2n){margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(odd){clear:left}.wy-control-group .wy-form-thirds{float:left;display:block;margin-right:2.35765%;width:31.76157%}.wy-control-group .wy-form-thirds:last-child,.wy-control-group .wy-form-thirds:nth-of-type(3n){margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n+1){clear:left}.wy-control-group.wy-control-group-no-input .wy-control,.wy-control-no-input{margin:6px 0 0;font-size:90%}.wy-control-no-input{display:inline-block}.wy-control-group.fluid-input input[type=color],.wy-control-group.fluid-input input[type=date],.wy-control-group.fluid-input input[type=datetime-local],.wy-control-group.fluid-input input[type=datetime],.wy-control-group.fluid-input input[type=email],.wy-control-group.fluid-input input[type=month],.wy-control-group.fluid-input input[type=number],.wy-control-group.fluid-input input[type=password],.wy-control-group.fluid-input input[type=search],.wy-control-group.fluid-input input[type=tel],.wy-control-group.fluid-input input[type=text],.wy-control-group.fluid-input input[type=time],.wy-control-group.fluid-input input[type=url],.wy-control-group.fluid-input input[type=week]{width:100%}.wy-form-message-inline{padding-left:.3em;color:#666;font-size:90%}.wy-form-message{display:block;color:#999;font-size:70%;margin-top:.3125em;font-style:italic}.wy-form-message p{font-size:inherit;font-style:italic;margin-bottom:6px}.wy-form-message p:last-child{margin-bottom:0}input{line-height:normal}input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;*overflow:visible}input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week]{-webkit-appearance:none;padding:6px;display:inline-block;border:1px solid #ccc;font-size:80%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 3px #ddd;border-radius:0;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}input[type=datetime-local]{padding:.34375em .625em}input[disabled]{cursor:default}input[type=checkbox],input[type=radio]{padding:0;margin-right:.3125em;*height:13px;*width:13px}input[type=checkbox],input[type=radio],input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}input[type=color]:focus,input[type=date]:focus,input[type=datetime-local]:focus,input[type=datetime]:focus,input[type=email]:focus,input[type=month]:focus,input[type=number]:focus,input[type=password]:focus,input[type=search]:focus,input[type=tel]:focus,input[type=text]:focus,input[type=time]:focus,input[type=url]:focus,input[type=week]:focus{outline:0;outline:thin dotted\9;border-color:#333}input.no-focus:focus{border-color:#ccc!important}input[type=checkbox]:focus,input[type=file]:focus,input[type=radio]:focus{outline:thin dotted #333;outline:1px auto #129fea}input[type=color][disabled],input[type=date][disabled],input[type=datetime-local][disabled],input[type=datetime][disabled],input[type=email][disabled],input[type=month][disabled],input[type=number][disabled],input[type=password][disabled],input[type=search][disabled],input[type=tel][disabled],input[type=text][disabled],input[type=time][disabled],input[type=url][disabled],input[type=week][disabled]{cursor:not-allowed;background-color:#fafafa}input:focus:invalid,select:focus:invalid,textarea:focus:invalid{color:#e74c3c;border:1px solid #e74c3c}input:focus:invalid:focus,select:focus:invalid:focus,textarea:focus:invalid:focus{border-color:#e74c3c}input[type=checkbox]:focus:invalid:focus,input[type=file]:focus:invalid:focus,input[type=radio]:focus:invalid:focus{outline-color:#e74c3c}input.wy-input-large{padding:12px;font-size:100%}textarea{overflow:auto;vertical-align:top;width:100%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif}select,textarea{padding:.5em .625em;display:inline-block;border:1px solid #ccc;font-size:80%;box-shadow:inset 0 1px 3px #ddd;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}select{border:1px solid #ccc;background-color:#fff}select[multiple]{height:auto}select:focus,textarea:focus{outline:0}input[readonly],select[disabled],select[readonly],textarea[disabled],textarea[readonly]{cursor:not-allowed;background-color:#fafafa}input[type=checkbox][disabled],input[type=radio][disabled]{cursor:not-allowed}.wy-checkbox,.wy-radio{margin:6px 0;color:#404040;display:block}.wy-checkbox input,.wy-radio input{vertical-align:baseline}.wy-form-message-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-input-prefix,.wy-input-suffix{white-space:nowrap;padding:6px}.wy-input-prefix .wy-input-context,.wy-input-suffix .wy-input-context{line-height:27px;padding:0 8px;display:inline-block;font-size:80%;background-color:#f3f6f6;border:1px solid #ccc;color:#999}.wy-input-suffix .wy-input-context{border-left:0}.wy-input-prefix .wy-input-context{border-right:0}.wy-switch{position:relative;display:block;height:24px;margin-top:12px;cursor:pointer}.wy-switch:before{left:0;top:0;width:36px;height:12px;background:#ccc}.wy-switch:after,.wy-switch:before{position:absolute;content:"";display:block;border-radius:4px;-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.wy-switch:after{width:18px;height:18px;background:#999;left:-3px;top:-3px}.wy-switch span{position:absolute;left:48px;display:block;font-size:12px;color:#ccc;line-height:1}.wy-switch.active:before{background:#1e8449}.wy-switch.active:after{left:24px;background:#27ae60}.wy-switch.disabled{cursor:not-allowed;opacity:.8}.wy-control-group.wy-control-group-error .wy-form-message,.wy-control-group.wy-control-group-error>label{color:#e74c3c}.wy-control-group.wy-control-group-error input[type=color],.wy-control-group.wy-control-group-error input[type=date],.wy-control-group.wy-control-group-error input[type=datetime-local],.wy-control-group.wy-control-group-error input[type=datetime],.wy-control-group.wy-control-group-error input[type=email],.wy-control-group.wy-control-group-error input[type=month],.wy-control-group.wy-control-group-error input[type=number],.wy-control-group.wy-control-group-error input[type=password],.wy-control-group.wy-control-group-error input[type=search],.wy-control-group.wy-control-group-error input[type=tel],.wy-control-group.wy-control-group-error input[type=text],.wy-control-group.wy-control-group-error input[type=time],.wy-control-group.wy-control-group-error input[type=url],.wy-control-group.wy-control-group-error input[type=week],.wy-control-group.wy-control-group-error textarea{border:1px solid #e74c3c}.wy-inline-validate{white-space:nowrap}.wy-inline-validate .wy-input-context{padding:.5em .625em;display:inline-block;font-size:80%}.wy-inline-validate.wy-inline-validate-success .wy-input-context{color:#27ae60}.wy-inline-validate.wy-inline-validate-danger .wy-input-context{color:#e74c3c}.wy-inline-validate.wy-inline-validate-warning .wy-input-context{color:#e67e22}.wy-inline-validate.wy-inline-validate-info .wy-input-context{color:#2980b9}.rotate-90{-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.rotate-180{-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.rotate-270{-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.mirror{-webkit-transform:scaleX(-1);-moz-transform:scaleX(-1);-ms-transform:scaleX(-1);-o-transform:scaleX(-1);transform:scaleX(-1)}.mirror.rotate-90{-webkit-transform:scaleX(-1) rotate(90deg);-moz-transform:scaleX(-1) rotate(90deg);-ms-transform:scaleX(-1) rotate(90deg);-o-transform:scaleX(-1) rotate(90deg);transform:scaleX(-1) rotate(90deg)}.mirror.rotate-180{-webkit-transform:scaleX(-1) rotate(180deg);-moz-transform:scaleX(-1) rotate(180deg);-ms-transform:scaleX(-1) rotate(180deg);-o-transform:scaleX(-1) rotate(180deg);transform:scaleX(-1) rotate(180deg)}.mirror.rotate-270{-webkit-transform:scaleX(-1) rotate(270deg);-moz-transform:scaleX(-1) rotate(270deg);-ms-transform:scaleX(-1) rotate(270deg);-o-transform:scaleX(-1) rotate(270deg);transform:scaleX(-1) rotate(270deg)}@media only screen and (max-width:480px){.wy-form button[type=submit]{margin:.7em 0 0}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=text],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week],.wy-form label{margin-bottom:.3em;display:block}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week]{margin-bottom:0}.wy-form-aligned .wy-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.wy-form-aligned .wy-control{margin:1.5em 0 0}.wy-form-message,.wy-form-message-inline,.wy-form .wy-help-inline{display:block;font-size:80%;padding:6px 0}}@media screen and (max-width:768px){.tablet-hide{display:none}}@media screen and (max-width:480px){.mobile-hide{display:none}}.float-left{float:left}.float-right{float:right}.full-width{width:100%}.rst-content table.docutils,.rst-content table.field-list,.wy-table{border-collapse:collapse;border-spacing:0;empty-cells:show;margin-bottom:24px}.rst-content table.docutils caption,.rst-content table.field-list caption,.wy-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.rst-content table.docutils td,.rst-content table.docutils th,.rst-content table.field-list td,.rst-content table.field-list th,.wy-table td,.wy-table th{font-size:90%;margin:0;overflow:visible;padding:8px 16px}.rst-content table.docutils td:first-child,.rst-content table.docutils th:first-child,.rst-content table.field-list td:first-child,.rst-content table.field-list th:first-child,.wy-table td:first-child,.wy-table th:first-child{border-left-width:0}.rst-content table.docutils thead,.rst-content table.field-list thead,.wy-table thead{color:#000;text-align:left;vertical-align:bottom;white-space:nowrap}.rst-content table.docutils thead th,.rst-content table.field-list thead th,.wy-table thead th{font-weight:700;border-bottom:2px solid #e1e4e5}.rst-content table.docutils td,.rst-content table.field-list td,.wy-table td{background-color:transparent;vertical-align:middle}.rst-content table.docutils td p,.rst-content table.field-list td p,.wy-table td p{line-height:18px}.rst-content table.docutils td p:last-child,.rst-content table.field-list td p:last-child,.wy-table td p:last-child{margin-bottom:0}.rst-content table.docutils .wy-table-cell-min,.rst-content table.field-list .wy-table-cell-min,.wy-table .wy-table-cell-min{width:1%;padding-right:0}.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox],.wy-table .wy-table-cell-min input[type=checkbox]{margin:0}.wy-table-secondary{color:grey;font-size:90%}.wy-table-tertiary{color:grey;font-size:80%}.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,.wy-table-backed,.wy-table-odd td,.wy-table-striped tr:nth-child(2n-1) td{background-color:#f3f6f6}.rst-content table.docutils,.wy-table-bordered-all{border:1px solid #e1e4e5}.rst-content table.docutils td,.wy-table-bordered-all td{border-bottom:1px solid #e1e4e5;border-left:1px solid #e1e4e5}.rst-content table.docutils tbody>tr:last-child td,.wy-table-bordered-all tbody>tr:last-child td{border-bottom-width:0}.wy-table-bordered{border:1px solid #e1e4e5}.wy-table-bordered-rows td{border-bottom:1px solid #e1e4e5}.wy-table-bordered-rows tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal td,.wy-table-horizontal th{border-width:0 0 1px;border-bottom:1px solid #e1e4e5}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-responsive{margin-bottom:24px;max-width:100%;overflow:auto}.wy-table-responsive table{margin-bottom:0!important}.wy-table-responsive table td,.wy-table-responsive table th{white-space:nowrap}a{color:#2980b9;text-decoration:none;cursor:pointer}a:hover{color:#3091d1}a:visited{color:#9b59b6}html{height:100%}body,html{overflow-x:hidden}body{font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-weight:400;color:#404040;min-height:100%;background:#edf0f2}.wy-text-left{text-align:left}.wy-text-center{text-align:center}.wy-text-right{text-align:right}.wy-text-large{font-size:120%}.wy-text-normal{font-size:100%}.wy-text-small,small{font-size:80%}.wy-text-strike{text-decoration:line-through}.wy-text-warning{color:#e67e22!important}a.wy-text-warning:hover{color:#eb9950!important}.wy-text-info{color:#2980b9!important}a.wy-text-info:hover{color:#409ad5!important}.wy-text-success{color:#27ae60!important}a.wy-text-success:hover{color:#36d278!important}.wy-text-danger{color:#e74c3c!important}a.wy-text-danger:hover{color:#ed7669!important}.wy-text-neutral{color:#404040!important}a.wy-text-neutral:hover{color:#595959!important}.rst-content .toctree-wrapper>p.caption,h1,h2,h3,h4,h5,h6,legend{margin-top:0;font-weight:700;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif}p{line-height:24px;font-size:16px;margin:0 0 24px}h1{font-size:175%}.rst-content .toctree-wrapper>p.caption,h2{font-size:150%}h3{font-size:125%}h4{font-size:115%}h5{font-size:110%}h6{font-size:100%}hr{display:block;height:1px;border:0;border-top:1px solid #e1e4e5;margin:24px 0;padding:0}.rst-content code,.rst-content tt,code{white-space:nowrap;max-width:100%;background:#fff;border:1px solid #e1e4e5;font-size:75%;padding:0 5px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#e74c3c;overflow-x:auto}.rst-content tt.code-large,code.code-large{font-size:90%}.rst-content .section ul,.rst-content .toctree-wrapper ul,.rst-content section ul,.wy-plain-list-disc,article ul{list-style:disc;line-height:24px;margin-bottom:24px}.rst-content .section ul li,.rst-content .toctree-wrapper ul li,.rst-content section ul li,.wy-plain-list-disc li,article ul li{list-style:disc;margin-left:24px}.rst-content .section ul li p:last-child,.rst-content .section ul li ul,.rst-content .toctree-wrapper ul li p:last-child,.rst-content .toctree-wrapper ul li ul,.rst-content section ul li p:last-child,.rst-content section ul li ul,.wy-plain-list-disc li p:last-child,.wy-plain-list-disc li ul,article ul li p:last-child,article ul li ul{margin-bottom:0}.rst-content .section ul li li,.rst-content .toctree-wrapper ul li li,.rst-content section ul li li,.wy-plain-list-disc li li,article ul li li{list-style:circle}.rst-content .section ul li li li,.rst-content .toctree-wrapper ul li li li,.rst-content section ul li li li,.wy-plain-list-disc li li li,article ul li li li{list-style:square}.rst-content .section ul li ol li,.rst-content .toctree-wrapper ul li ol li,.rst-content section ul li ol li,.wy-plain-list-disc li ol li,article ul li ol li{list-style:decimal}.rst-content .section ol,.rst-content .section ol.arabic,.rst-content .toctree-wrapper ol,.rst-content .toctree-wrapper ol.arabic,.rst-content section ol,.rst-content section ol.arabic,.wy-plain-list-decimal,article ol{list-style:decimal;line-height:24px;margin-bottom:24px}.rst-content .section ol.arabic li,.rst-content .section ol li,.rst-content .toctree-wrapper ol.arabic li,.rst-content .toctree-wrapper ol li,.rst-content section ol.arabic li,.rst-content section ol li,.wy-plain-list-decimal li,article ol li{list-style:decimal;margin-left:24px}.rst-content .section ol.arabic li ul,.rst-content .section ol li p:last-child,.rst-content .section ol li ul,.rst-content .toctree-wrapper ol.arabic li ul,.rst-content .toctree-wrapper ol li p:last-child,.rst-content .toctree-wrapper ol li ul,.rst-content section ol.arabic li ul,.rst-content section ol li p:last-child,.rst-content section ol li ul,.wy-plain-list-decimal li p:last-child,.wy-plain-list-decimal li ul,article ol li p:last-child,article ol li ul{margin-bottom:0}.rst-content .section ol.arabic li ul li,.rst-content .section ol li ul li,.rst-content .toctree-wrapper ol.arabic li ul li,.rst-content .toctree-wrapper ol li ul li,.rst-content section ol.arabic li ul li,.rst-content section ol li ul li,.wy-plain-list-decimal li ul li,article ol li ul li{list-style:disc}.wy-breadcrumbs{*zoom:1}.wy-breadcrumbs:after,.wy-breadcrumbs:before{display:table;content:""}.wy-breadcrumbs:after{clear:both}.wy-breadcrumbs>li{display:inline-block;padding-top:5px}.wy-breadcrumbs>li.wy-breadcrumbs-aside{float:right}.rst-content .wy-breadcrumbs>li code,.rst-content .wy-breadcrumbs>li tt,.wy-breadcrumbs>li .rst-content tt,.wy-breadcrumbs>li code{all:inherit;color:inherit}.breadcrumb-item:before{content:"/";color:#bbb;font-size:13px;padding:0 6px 0 3px}.wy-breadcrumbs-extra{margin-bottom:0;color:#b3b3b3;font-size:80%;display:inline-block}@media screen and (max-width:480px){.wy-breadcrumbs-extra,.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}@media print{.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}html{font-size:16px}.wy-affix{position:fixed;top:1.618em}.wy-menu a:hover{text-decoration:none}.wy-menu-horiz{*zoom:1}.wy-menu-horiz:after,.wy-menu-horiz:before{display:table;content:""}.wy-menu-horiz:after{clear:both}.wy-menu-horiz li,.wy-menu-horiz ul{display:inline-block}.wy-menu-horiz li:hover{background:hsla(0,0%,100%,.1)}.wy-menu-horiz li.divide-left{border-left:1px solid #404040}.wy-menu-horiz li.divide-right{border-right:1px solid #404040}.wy-menu-horiz a{height:32px;display:inline-block;line-height:32px;padding:0 16px}.wy-menu-vertical{width:300px}.wy-menu-vertical header,.wy-menu-vertical p.caption{color:#55a5d9;height:32px;line-height:32px;padding:0 1.618em;margin:12px 0 0;display:block;font-weight:700;text-transform:uppercase;font-size:85%;white-space:nowrap}.wy-menu-vertical ul{margin-bottom:0}.wy-menu-vertical li.divide-top{border-top:1px solid #404040}.wy-menu-vertical li.divide-bottom{border-bottom:1px solid #404040}.wy-menu-vertical li.current{background:#e3e3e3}.wy-menu-vertical li.current a{color:grey;border-right:1px solid #c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.current a:hover{background:#d6d6d6}.rst-content .wy-menu-vertical li tt,.wy-menu-vertical li .rst-content tt,.wy-menu-vertical li code{border:none;background:inherit;color:inherit;padding-left:0;padding-right:0}.wy-menu-vertical li button.toctree-expand{display:block;float:left;margin-left:-1.2em;line-height:18px;color:#4d4d4d;border:none;background:none;padding:0}.wy-menu-vertical li.current>a,.wy-menu-vertical li.on a{color:#404040;font-weight:700;position:relative;background:#fcfcfc;border:none;padding:.4045em 1.618em}.wy-menu-vertical li.current>a:hover,.wy-menu-vertical li.on a:hover{background:#fcfcfc}.wy-menu-vertical li.current>a:hover button.toctree-expand,.wy-menu-vertical li.on a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand{display:block;line-height:18px;color:#333}.wy-menu-vertical li.toctree-l1.current>a{border-bottom:1px solid #c9c9c9;border-top:1px solid #c9c9c9}.wy-menu-vertical .toctree-l1.current .toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .toctree-l11>ul{display:none}.wy-menu-vertical .toctree-l1.current .current.toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .current.toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .current.toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .current.toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .current.toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .current.toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .current.toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .current.toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .current.toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .current.toctree-l11>ul{display:block}.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4{font-size:.9em}.wy-menu-vertical li.toctree-l2 a,.wy-menu-vertical li.toctree-l3 a,.wy-menu-vertical li.toctree-l4 a,.wy-menu-vertical li.toctree-l5 a,.wy-menu-vertical li.toctree-l6 a,.wy-menu-vertical li.toctree-l7 a,.wy-menu-vertical li.toctree-l8 a,.wy-menu-vertical li.toctree-l9 a,.wy-menu-vertical li.toctree-l10 a{color:#404040}.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l4 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l5 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l6 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l7 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l8 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l9 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l10 a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a,.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a,.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a,.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a,.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a,.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a,.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a,.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{display:block}.wy-menu-vertical li.toctree-l2.current>a{padding:.4045em 2.427em}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{padding:.4045em 1.618em .4045em 4.045em}.wy-menu-vertical li.toctree-l3.current>a{padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{padding:.4045em 1.618em .4045em 5.663em}.wy-menu-vertical li.toctree-l4.current>a{padding:.4045em 5.663em}.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a{padding:.4045em 1.618em .4045em 7.281em}.wy-menu-vertical li.toctree-l5.current>a{padding:.4045em 7.281em}.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a{padding:.4045em 1.618em .4045em 8.899em}.wy-menu-vertical li.toctree-l6.current>a{padding:.4045em 8.899em}.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a{padding:.4045em 1.618em .4045em 10.517em}.wy-menu-vertical li.toctree-l7.current>a{padding:.4045em 10.517em}.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a{padding:.4045em 1.618em .4045em 12.135em}.wy-menu-vertical li.toctree-l8.current>a{padding:.4045em 12.135em}.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a{padding:.4045em 1.618em .4045em 13.753em}.wy-menu-vertical li.toctree-l9.current>a{padding:.4045em 13.753em}.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a{padding:.4045em 1.618em .4045em 15.371em}.wy-menu-vertical li.toctree-l10.current>a{padding:.4045em 15.371em}.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{padding:.4045em 1.618em .4045em 16.989em}.wy-menu-vertical li.toctree-l2.current>a,.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{background:#c9c9c9}.wy-menu-vertical li.toctree-l2 button.toctree-expand{color:#a3a3a3}.wy-menu-vertical li.toctree-l3.current>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{background:#bdbdbd}.wy-menu-vertical li.toctree-l3 button.toctree-expand{color:#969696}.wy-menu-vertical li.current ul{display:block}.wy-menu-vertical li ul{margin-bottom:0;display:none}.wy-menu-vertical li ul li a{margin-bottom:0;color:#d9d9d9;font-weight:400}.wy-menu-vertical a{line-height:18px;padding:.4045em 1.618em;display:block;position:relative;font-size:90%;color:#d9d9d9}.wy-menu-vertical a:hover{background-color:#4e4a4a;cursor:pointer}.wy-menu-vertical a:hover button.toctree-expand{color:#d9d9d9}.wy-menu-vertical a:active{background-color:#2980b9;cursor:pointer;color:#fff}.wy-menu-vertical a:active button.toctree-expand{color:#fff}.wy-side-nav-search{display:block;width:300px;padding:.809em;margin-bottom:.809em;z-index:200;background-color:#2980b9;text-align:center;color:#fcfcfc}.wy-side-nav-search input[type=text]{width:100%;border-radius:50px;padding:6px 12px;border-color:#2472a4}.wy-side-nav-search img{display:block;margin:auto auto .809em;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-side-nav-search .wy-dropdown>a,.wy-side-nav-search>a{color:#fcfcfc;font-size:100%;font-weight:700;display:inline-block;padding:4px 6px;margin-bottom:.809em;max-width:100%}.wy-side-nav-search .wy-dropdown>a:hover,.wy-side-nav-search>a:hover{background:hsla(0,0%,100%,.1)}.wy-side-nav-search .wy-dropdown>a img.logo,.wy-side-nav-search>a img.logo{display:block;margin:0 auto;height:auto;width:auto;border-radius:0;max-width:100%;background:transparent}.wy-side-nav-search .wy-dropdown>a.icon img.logo,.wy-side-nav-search>a.icon img.logo{margin-top:.85em}.wy-side-nav-search>div.version{margin-top:-.4045em;margin-bottom:.809em;font-weight:400;color:hsla(0,0%,100%,.3)}.wy-nav .wy-menu-vertical header{color:#2980b9}.wy-nav .wy-menu-vertical a{color:#b3b3b3}.wy-nav .wy-menu-vertical a:hover{background-color:#2980b9;color:#fff}[data-menu-wrap]{-webkit-transition:all .2s ease-in;-moz-transition:all .2s ease-in;transition:all .2s ease-in;position:absolute;opacity:1;width:100%;opacity:0}[data-menu-wrap].move-center{left:0;right:auto;opacity:1}[data-menu-wrap].move-left{right:auto;left:-100%;opacity:0}[data-menu-wrap].move-right{right:-100%;left:auto;opacity:0}.wy-body-for-nav{background:#fcfcfc}.wy-grid-for-nav{position:absolute;width:100%;height:100%}.wy-nav-side{position:fixed;top:0;bottom:0;left:0;padding-bottom:2em;width:300px;overflow-x:hidden;overflow-y:hidden;min-height:100%;color:#9b9b9b;background:#343131;z-index:200}.wy-side-scroll{width:320px;position:relative;overflow-x:hidden;overflow-y:scroll;height:100%}.wy-nav-top{display:none;background:#2980b9;color:#fff;padding:.4045em .809em;position:relative;line-height:50px;text-align:center;font-size:100%;*zoom:1}.wy-nav-top:after,.wy-nav-top:before{display:table;content:""}.wy-nav-top:after{clear:both}.wy-nav-top a{color:#fff;font-weight:700}.wy-nav-top img{margin-right:12px;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-nav-top i{font-size:30px;float:left;cursor:pointer;padding-top:inherit}.wy-nav-content-wrap{margin-left:300px;background:#fcfcfc;min-height:100%}.wy-nav-content{padding:1.618em 3.236em;height:100%;max-width:800px;margin:auto}.wy-body-mask{position:fixed;width:100%;height:100%;background:rgba(0,0,0,.2);display:none;z-index:499}.wy-body-mask.on{display:block}footer{color:grey}footer p{margin-bottom:12px}.rst-content footer span.commit tt,footer span.commit .rst-content tt,footer span.commit code{padding:0;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:1em;background:none;border:none;color:grey}.rst-footer-buttons{*zoom:1}.rst-footer-buttons:after,.rst-footer-buttons:before{width:100%;display:table;content:""}.rst-footer-buttons:after{clear:both}.rst-breadcrumbs-buttons{margin-top:12px;*zoom:1}.rst-breadcrumbs-buttons:after,.rst-breadcrumbs-buttons:before{display:table;content:""}.rst-breadcrumbs-buttons:after{clear:both}#search-results .search li{margin-bottom:24px;border-bottom:1px solid #e1e4e5;padding-bottom:24px}#search-results .search li:first-child{border-top:1px solid #e1e4e5;padding-top:24px}#search-results .search li a{font-size:120%;margin-bottom:12px;display:inline-block}#search-results .context{color:grey;font-size:90%}.genindextable li>ul{margin-left:24px}@media screen and (max-width:768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}.wy-nav-side{left:-300px}.wy-nav-side.shift{width:85%;left:0}.wy-menu.wy-menu-vertical,.wy-side-nav-search,.wy-side-scroll{width:auto}.wy-nav-content-wrap{margin-left:0}.wy-nav-content-wrap .wy-nav-content{padding:1.618em}.wy-nav-content-wrap.shift{position:fixed;min-width:100%;left:85%;top:0;height:100%;overflow:hidden}}@media screen and (min-width:1100px){.wy-nav-content-wrap{background:rgba(0,0,0,.05)}.wy-nav-content{margin:0;background:#fcfcfc}}@media print{.rst-versions,.wy-nav-side,footer{display:none}.wy-nav-content-wrap{margin-left:0}}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60;*zoom:1}.rst-versions .rst-current-version:after,.rst-versions .rst-current-version:before{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-content .code-block-caption .rst-versions .rst-current-version .headerlink,.rst-content .eqno .rst-versions .rst-current-version .headerlink,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-content code.download .rst-versions .rst-current-version span:first-child,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-content p .rst-versions .rst-current-version .headerlink,.rst-content table>caption .rst-versions .rst-current-version .headerlink,.rst-content tt.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .icon,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-versions .rst-current-version .rst-content .code-block-caption .headerlink,.rst-versions .rst-current-version .rst-content .eqno .headerlink,.rst-versions .rst-current-version .rst-content code.download span:first-child,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-versions .rst-current-version .rst-content p .headerlink,.rst-versions .rst-current-version .rst-content table>caption .headerlink,.rst-versions .rst-current-version .rst-content tt.download span:first-child,.rst-versions .rst-current-version .wy-menu-vertical li button.toctree-expand,.wy-menu-vertical li .rst-versions .rst-current-version button.toctree-expand{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}.rst-content .toctree-wrapper>p.caption,.rst-content h1,.rst-content h2,.rst-content h3,.rst-content h4,.rst-content h5,.rst-content h6{margin-bottom:24px}.rst-content img{max-width:100%;height:auto}.rst-content div.figure,.rst-content figure{margin-bottom:24px}.rst-content div.figure .caption-text,.rst-content figure .caption-text{font-style:italic}.rst-content div.figure p:last-child.caption,.rst-content figure p:last-child.caption{margin-bottom:0}.rst-content div.figure.align-center,.rst-content figure.align-center{text-align:center}.rst-content .section>a>img,.rst-content .section>img,.rst-content section>a>img,.rst-content section>img{margin-bottom:24px}.rst-content abbr[title]{text-decoration:none}.rst-content.style-external-links a.reference.external:after{font-family:FontAwesome;content:"\f08e";color:#b3b3b3;vertical-align:super;font-size:60%;margin:0 .2em}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content pre.literal-block{white-space:pre;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;display:block;overflow:auto}.rst-content div[class^=highlight],.rst-content pre.literal-block{border:1px solid #e1e4e5;overflow-x:auto;margin:1px 0 24px}.rst-content div[class^=highlight] div[class^=highlight],.rst-content pre.literal-block div[class^=highlight]{padding:0;border:none;margin:0}.rst-content div[class^=highlight] td.code{width:100%}.rst-content .linenodiv pre{border-right:1px solid #e6e9ea;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;user-select:none;pointer-events:none}.rst-content div[class^=highlight] pre{white-space:pre;margin:0;padding:12px;display:block;overflow:auto}.rst-content div[class^=highlight] pre .hll{display:block;margin:0 -12px;padding:0 12px}.rst-content .linenodiv pre,.rst-content div[class^=highlight] pre,.rst-content pre.literal-block{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:12px;line-height:1.4}.rst-content div.highlight .gp,.rst-content div.highlight span.linenos{user-select:none;pointer-events:none}.rst-content div.highlight span.linenos{display:inline-block;padding-left:0;padding-right:12px;margin-right:12px;border-right:1px solid #e6e9ea}.rst-content .code-block-caption{font-style:italic;font-size:85%;line-height:1;padding:1em 0;text-align:center}@media print{.rst-content .codeblock,.rst-content div[class^=highlight],.rst-content div[class^=highlight] pre{white-space:pre-wrap}}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning{clear:both}.rst-content .admonition-todo .last,.rst-content .admonition-todo>:last-child,.rst-content .admonition .last,.rst-content .admonition>:last-child,.rst-content .attention .last,.rst-content .attention>:last-child,.rst-content .caution .last,.rst-content .caution>:last-child,.rst-content .danger .last,.rst-content .danger>:last-child,.rst-content .error .last,.rst-content .error>:last-child,.rst-content .hint .last,.rst-content .hint>:last-child,.rst-content .important .last,.rst-content .important>:last-child,.rst-content .note .last,.rst-content .note>:last-child,.rst-content .seealso .last,.rst-content .seealso>:last-child,.rst-content .tip .last,.rst-content .tip>:last-child,.rst-content .warning .last,.rst-content .warning>:last-child{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent!important;border-color:rgba(0,0,0,.1)!important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha>li,.rst-content .toctree-wrapper ol.loweralpha,.rst-content .toctree-wrapper ol.loweralpha>li,.rst-content section ol.loweralpha,.rst-content section ol.loweralpha>li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha>li,.rst-content .toctree-wrapper ol.upperalpha,.rst-content .toctree-wrapper ol.upperalpha>li,.rst-content section ol.upperalpha,.rst-content section ol.upperalpha>li{list-style:upper-alpha}.rst-content .section ol li>*,.rst-content .section ul li>*,.rst-content .toctree-wrapper ol li>*,.rst-content .toctree-wrapper ul li>*,.rst-content section ol li>*,.rst-content section ul li>*{margin-top:12px;margin-bottom:12px}.rst-content .section ol li>:first-child,.rst-content .section ul li>:first-child,.rst-content .toctree-wrapper ol li>:first-child,.rst-content .toctree-wrapper ul li>:first-child,.rst-content section ol li>:first-child,.rst-content section ul li>:first-child{margin-top:0}.rst-content .section ol li>p,.rst-content .section ol li>p:last-child,.rst-content .section ul li>p,.rst-content .section ul li>p:last-child,.rst-content .toctree-wrapper ol li>p,.rst-content .toctree-wrapper ol li>p:last-child,.rst-content .toctree-wrapper ul li>p,.rst-content .toctree-wrapper ul li>p:last-child,.rst-content section ol li>p,.rst-content section ol li>p:last-child,.rst-content section ul li>p,.rst-content section ul li>p:last-child{margin-bottom:12px}.rst-content .section ol li>p:only-child,.rst-content .section ol li>p:only-child:last-child,.rst-content .section ul li>p:only-child,.rst-content .section ul li>p:only-child:last-child,.rst-content .toctree-wrapper ol li>p:only-child,.rst-content .toctree-wrapper ol li>p:only-child:last-child,.rst-content .toctree-wrapper ul li>p:only-child,.rst-content .toctree-wrapper ul li>p:only-child:last-child,.rst-content section ol li>p:only-child,.rst-content section ol li>p:only-child:last-child,.rst-content section ul li>p:only-child,.rst-content section ul li>p:only-child:last-child{margin-bottom:0}.rst-content .section ol li>ol,.rst-content .section ol li>ul,.rst-content .section ul li>ol,.rst-content .section ul li>ul,.rst-content .toctree-wrapper ol li>ol,.rst-content .toctree-wrapper ol li>ul,.rst-content .toctree-wrapper ul li>ol,.rst-content .toctree-wrapper ul li>ul,.rst-content section ol li>ol,.rst-content section ol li>ul,.rst-content section ul li>ol,.rst-content section ul li>ul{margin-bottom:12px}.rst-content .section ol.simple li>*,.rst-content .section ol.simple li ol,.rst-content .section ol.simple li ul,.rst-content .section ul.simple li>*,.rst-content .section ul.simple li ol,.rst-content .section ul.simple li ul,.rst-content .toctree-wrapper ol.simple li>*,.rst-content .toctree-wrapper ol.simple li ol,.rst-content .toctree-wrapper ol.simple li ul,.rst-content .toctree-wrapper ul.simple li>*,.rst-content .toctree-wrapper ul.simple li ol,.rst-content .toctree-wrapper ul.simple li ul,.rst-content section ol.simple li>*,.rst-content section ol.simple li ol,.rst-content section ol.simple li ul,.rst-content section ul.simple li>*,.rst-content section ul.simple li ol,.rst-content section ul.simple li ul{margin-top:0;margin-bottom:0}.rst-content .line-block{margin-left:0;margin-bottom:24px;line-height:24px}.rst-content .line-block .line-block{margin-left:24px;margin-bottom:0}.rst-content .topic-title{font-weight:700;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0 0 24px 24px}.rst-content .align-left{float:left;margin:0 24px 24px 0}.rst-content .align-center{margin:auto}.rst-content .align-center:not(table){display:block}.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink{opacity:0;font-size:14px;font-family:FontAwesome;margin-left:.5em}.rst-content .code-block-caption .headerlink:focus,.rst-content .code-block-caption:hover .headerlink,.rst-content .eqno .headerlink:focus,.rst-content .eqno:hover .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink:focus,.rst-content .toctree-wrapper>p.caption:hover .headerlink,.rst-content dl dt .headerlink:focus,.rst-content dl dt:hover .headerlink,.rst-content h1 .headerlink:focus,.rst-content h1:hover .headerlink,.rst-content h2 .headerlink:focus,.rst-content h2:hover .headerlink,.rst-content h3 .headerlink:focus,.rst-content h3:hover .headerlink,.rst-content h4 .headerlink:focus,.rst-content h4:hover .headerlink,.rst-content h5 .headerlink:focus,.rst-content h5:hover .headerlink,.rst-content h6 .headerlink:focus,.rst-content h6:hover .headerlink,.rst-content p.caption .headerlink:focus,.rst-content p.caption:hover .headerlink,.rst-content p .headerlink:focus,.rst-content p:hover .headerlink,.rst-content table>caption .headerlink:focus,.rst-content table>caption:hover .headerlink{opacity:1}.rst-content p a{overflow-wrap:anywhere}.rst-content .wy-table td p,.rst-content .wy-table td ul,.rst-content .wy-table th p,.rst-content .wy-table th ul,.rst-content table.docutils td p,.rst-content table.docutils td ul,.rst-content table.docutils th p,.rst-content table.docutils th ul,.rst-content table.field-list td p,.rst-content table.field-list td ul,.rst-content table.field-list th p,.rst-content table.field-list th ul{font-size:inherit}.rst-content .btn:focus{outline:2px solid}.rst-content table>caption .headerlink:after{font-size:12px}.rst-content .centered{text-align:center}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:1px solid #e1e4e5}.rst-content .sidebar dl,.rst-content .sidebar p,.rst-content .sidebar ul{font-size:90%}.rst-content .sidebar .last,.rst-content .sidebar>:last-child{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif;font-weight:700;background:#e1e4e5;padding:6px 12px;margin:-24px -24px 24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;box-shadow:0 0 0 2px #f1c40f;display:inline;font-weight:700}.rst-content .citation-reference,.rst-content .footnote-reference{vertical-align:baseline;position:relative;top:-.4em;line-height:0;font-size:90%}.rst-content .citation-reference>span.fn-bracket,.rst-content .footnote-reference>span.fn-bracket{display:none}.rst-content .hlist{width:100%}.rst-content dl dt span.classifier:before{content:" : "}.rst-content dl dt span.classifier-delimiter{display:none!important}html.writer-html4 .rst-content table.docutils.citation,html.writer-html4 .rst-content table.docutils.footnote{background:none;border:none}html.writer-html4 .rst-content table.docutils.citation td,html.writer-html4 .rst-content table.docutils.citation tr,html.writer-html4 .rst-content table.docutils.footnote td,html.writer-html4 .rst-content table.docutils.footnote tr{border:none;background-color:transparent!important;white-space:normal}html.writer-html4 .rst-content table.docutils.citation td.label,html.writer-html4 .rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{display:grid;grid-template-columns:auto minmax(80%,95%)}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{display:inline-grid;grid-template-columns:max-content auto}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{display:grid;grid-template-columns:auto auto minmax(.65rem,auto) minmax(40%,95%)}html.writer-html5 .rst-content aside.citation>span.label,html.writer-html5 .rst-content aside.footnote>span.label,html.writer-html5 .rst-content div.citation>span.label{grid-column-start:1;grid-column-end:2}html.writer-html5 .rst-content aside.citation>span.backrefs,html.writer-html5 .rst-content aside.footnote>span.backrefs,html.writer-html5 .rst-content div.citation>span.backrefs{grid-column-start:2;grid-column-end:3;grid-row-start:1;grid-row-end:3}html.writer-html5 .rst-content aside.citation>p,html.writer-html5 .rst-content aside.footnote>p,html.writer-html5 .rst-content div.citation>p{grid-column-start:4;grid-column-end:5}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{margin-bottom:24px}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{padding-left:1rem}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dd,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dd,html.writer-html5 .rst-content dl.footnote>dt{margin-bottom:0}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{font-size:.9rem}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.footnote>dt{margin:0 .5rem .5rem 0;line-height:1.2rem;word-break:break-all;font-weight:400}html.writer-html5 .rst-content dl.citation>dt>span.brackets:before,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:before{content:"["}html.writer-html5 .rst-content dl.citation>dt>span.brackets:after,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:after{content:"]"}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a{word-break:keep-all}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a:not(:first-child):before,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.footnote>dd{margin:0 0 .5rem;line-height:1.2rem}html.writer-html5 .rst-content dl.citation>dd p,html.writer-html5 .rst-content dl.footnote>dd p{font-size:.9rem}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{padding-left:1rem;padding-right:1rem;font-size:.9rem;line-height:1.2rem}html.writer-html5 .rst-content aside.citation p,html.writer-html5 .rst-content aside.footnote p,html.writer-html5 .rst-content div.citation p{font-size:.9rem;line-height:1.2rem;margin-bottom:12px}html.writer-html5 .rst-content aside.citation span.backrefs,html.writer-html5 .rst-content aside.footnote span.backrefs,html.writer-html5 .rst-content div.citation span.backrefs{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content aside.citation span.backrefs>a,html.writer-html5 .rst-content aside.footnote span.backrefs>a,html.writer-html5 .rst-content div.citation span.backrefs>a{word-break:keep-all}html.writer-html5 .rst-content aside.citation span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content aside.footnote span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content div.citation span.backrefs>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content aside.citation span.label,html.writer-html5 .rst-content aside.footnote span.label,html.writer-html5 .rst-content div.citation span.label{line-height:1.2rem}html.writer-html5 .rst-content aside.citation-list,html.writer-html5 .rst-content aside.footnote-list,html.writer-html5 .rst-content div.citation-list{margin-bottom:24px}html.writer-html5 .rst-content dl.option-list kbd{font-size:.9rem}.rst-content table.docutils.footnote,html.writer-html4 .rst-content table.docutils.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content aside.footnote-list aside.footnote,html.writer-html5 .rst-content div.citation-list>div.citation,html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{color:grey}.rst-content table.docutils.footnote code,.rst-content table.docutils.footnote tt,html.writer-html4 .rst-content table.docutils.citation code,html.writer-html4 .rst-content table.docutils.citation tt,html.writer-html5 .rst-content aside.footnote-list aside.footnote code,html.writer-html5 .rst-content aside.footnote-list aside.footnote tt,html.writer-html5 .rst-content aside.footnote code,html.writer-html5 .rst-content aside.footnote tt,html.writer-html5 .rst-content div.citation-list>div.citation code,html.writer-html5 .rst-content div.citation-list>div.citation tt,html.writer-html5 .rst-content dl.citation code,html.writer-html5 .rst-content dl.citation tt,html.writer-html5 .rst-content dl.footnote code,html.writer-html5 .rst-content dl.footnote tt{color:#555}.rst-content .wy-table-responsive.citation,.rst-content .wy-table-responsive.footnote{margin-bottom:0}.rst-content .wy-table-responsive.citation+:not(.citation),.rst-content .wy-table-responsive.footnote+:not(.footnote){margin-top:24px}.rst-content .wy-table-responsive.citation:last-child,.rst-content .wy-table-responsive.footnote:last-child{margin-bottom:24px}.rst-content table.docutils th{border-color:#e1e4e5}html.writer-html5 .rst-content table.docutils th{border:1px solid #e1e4e5}html.writer-html5 .rst-content table.docutils td>p,html.writer-html5 .rst-content table.docutils th>p{line-height:1rem;margin-bottom:0;font-size:.9rem}.rst-content table.docutils td .last,.rst-content table.docutils td .last>:last-child{margin-bottom:0}.rst-content table.field-list,.rst-content table.field-list td{border:none}.rst-content table.field-list td p{line-height:inherit}.rst-content table.field-list td>strong{display:inline-block}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left}.rst-content code,.rst-content tt{color:#000;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;padding:2px 5px}.rst-content code big,.rst-content code em,.rst-content tt big,.rst-content tt em{font-size:100%!important;line-height:normal}.rst-content code.literal,.rst-content tt.literal{color:#e74c3c;white-space:normal}.rst-content code.xref,.rst-content tt.xref,a .rst-content code,a .rst-content tt{font-weight:700;color:#404040;overflow-wrap:normal}.rst-content kbd,.rst-content pre,.rst-content samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace}.rst-content a code,.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:700;margin-bottom:12px}.rst-content dl ol,.rst-content dl p,.rst-content dl table,.rst-content dl ul{margin-bottom:12px}.rst-content dl dd{margin:0 0 12px 24px;line-height:24px}.rst-content dl dd>ol:last-child,.rst-content dl dd>p:last-child,.rst-content dl dd>table:last-child,.rst-content dl dd>ul:last-child{margin-bottom:0}html.writer-html4 .rst-content dl:not(.docutils),html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple){margin-bottom:24px}html.writer-html4 .rst-content dl:not(.docutils)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{display:table;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:3px solid #6ab0de;padding:6px;position:relative}html.writer-html4 .rst-content dl:not(.docutils)>dt:before,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:before{color:#6ab0de}html.writer-html4 .rst-content dl:not(.docutils)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{margin-bottom:6px;border:none;border-left:3px solid #ccc;background:#f0f0f0;color:#555}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils)>dt:first-child,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:first-child{margin-top:0}html.writer-html4 .rst-content dl:not(.docutils) code.descclassname,html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descclassname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{background-color:transparent;border:none;padding:0;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .optional,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .property,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .property{display:inline-block;padding-right:8px;max-width:100%}html.writer-html4 .rst-content dl:not(.docutils) .k,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .k{font-style:italic}html.writer-html4 .rst-content dl:not(.docutils) .descclassname,html.writer-html4 .rst-content dl:not(.docutils) .descname,html.writer-html4 .rst-content dl:not(.docutils) .sig-name,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .sig-name{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#000}.rst-content .viewcode-back,.rst-content .viewcode-link{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right}.rst-content p.rubric{margin-bottom:12px;font-weight:700}.rst-content code.download,.rst-content tt.download{background:inherit;padding:inherit;font-weight:400;font-family:inherit;font-size:inherit;color:inherit;border:inherit;white-space:inherit}.rst-content code.download span:first-child,.rst-content tt.download span:first-child{-webkit-font-smoothing:subpixel-antialiased}.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{margin-right:4px}.rst-content .guilabel,.rst-content .menuselection{font-size:80%;font-weight:700;border-radius:4px;padding:2.4px 6px;margin:auto 2px}.rst-content .guilabel,.rst-content .menuselection{border:1px solid #7fbbe3;background:#e7f2fa}.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>.kbd,.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>kbd{color:inherit;font-size:80%;background-color:#fff;border:1px solid #a6a6a6;border-radius:4px;box-shadow:0 2px grey;padding:2.4px 6px;margin:auto 0}.rst-content .versionmodified{font-style:italic}@media screen and (max-width:480px){.rst-content .sidebar{width:100%}}span[id*=MathJax-Span]{color:#404040}.math{text-align:center}@font-face{font-family:Lato;src:url(fonts/lato-normal.woff2?bd03a2cc277bbbc338d464e679fe9942) format("woff2"),url(fonts/lato-normal.woff?27bd77b9162d388cb8d4c4217c7c5e2a) format("woff");font-weight:400;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold.woff2?cccb897485813c7c256901dbca54ecf2) format("woff2"),url(fonts/lato-bold.woff?d878b6c29b10beca227e9eef4246111b) format("woff");font-weight:700;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold-italic.woff2?0b6bb6725576b072c5d0b02ecdd1900d) format("woff2"),url(fonts/lato-bold-italic.woff?9c7e4e9eb485b4a121c760e61bc3707c) format("woff");font-weight:700;font-style:italic;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-normal-italic.woff2?4eb103b4d12be57cb1d040ed5e162e9d) format("woff2"),url(fonts/lato-normal-italic.woff?f28f2d6482446544ef1ea1ccc6dd5892) format("woff");font-weight:400;font-style:italic;font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:400;src:url(fonts/Roboto-Slab-Regular.woff2?7abf5b8d04d26a2cafea937019bca958) format("woff2"),url(fonts/Roboto-Slab-Regular.woff?c1be9284088d487c5e3ff0a10a92e58c) format("woff");font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:700;src:url(fonts/Roboto-Slab-Bold.woff2?9984f4a9bda09be08e83f2506954adbe) format("woff2"),url(fonts/Roboto-Slab-Bold.woff?bed5564a116b05148e3b3bea6fb1162a) format("woff");font-display:block} \ No newline at end of file + */@font-face{font-family:FontAwesome;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713);src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix&v=4.7.0) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#fontawesomeregular) format("svg");font-weight:400;font-style:normal}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14286em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14286em;width:2.14286em;top:.14286em;text-align:center}.fa-li.fa-lg{left:-1.85714em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa-pull-left.icon,.fa.fa-pull-left,.rst-content .code-block-caption .fa-pull-left.headerlink,.rst-content .eqno .fa-pull-left.headerlink,.rst-content .fa-pull-left.admonition-title,.rst-content code.download span.fa-pull-left:first-child,.rst-content dl dt .fa-pull-left.headerlink,.rst-content h1 .fa-pull-left.headerlink,.rst-content h2 .fa-pull-left.headerlink,.rst-content h3 .fa-pull-left.headerlink,.rst-content h4 .fa-pull-left.headerlink,.rst-content h5 .fa-pull-left.headerlink,.rst-content h6 .fa-pull-left.headerlink,.rst-content p .fa-pull-left.headerlink,.rst-content table>caption .fa-pull-left.headerlink,.rst-content tt.download span.fa-pull-left:first-child,.wy-menu-vertical li.current>a button.fa-pull-left.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-left.toctree-expand,.wy-menu-vertical li button.fa-pull-left.toctree-expand{margin-right:.3em}.fa-pull-right.icon,.fa.fa-pull-right,.rst-content .code-block-caption .fa-pull-right.headerlink,.rst-content .eqno .fa-pull-right.headerlink,.rst-content .fa-pull-right.admonition-title,.rst-content code.download span.fa-pull-right:first-child,.rst-content dl dt .fa-pull-right.headerlink,.rst-content h1 .fa-pull-right.headerlink,.rst-content h2 .fa-pull-right.headerlink,.rst-content h3 .fa-pull-right.headerlink,.rst-content h4 .fa-pull-right.headerlink,.rst-content h5 .fa-pull-right.headerlink,.rst-content h6 .fa-pull-right.headerlink,.rst-content p .fa-pull-right.headerlink,.rst-content table>caption .fa-pull-right.headerlink,.rst-content tt.download span.fa-pull-right:first-child,.wy-menu-vertical li.current>a button.fa-pull-right.toctree-expand,.wy-menu-vertical li.on a button.fa-pull-right.toctree-expand,.wy-menu-vertical li button.fa-pull-right.toctree-expand{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left,.pull-left.icon,.rst-content .code-block-caption .pull-left.headerlink,.rst-content .eqno .pull-left.headerlink,.rst-content .pull-left.admonition-title,.rst-content code.download span.pull-left:first-child,.rst-content dl dt .pull-left.headerlink,.rst-content h1 .pull-left.headerlink,.rst-content h2 .pull-left.headerlink,.rst-content h3 .pull-left.headerlink,.rst-content h4 .pull-left.headerlink,.rst-content h5 .pull-left.headerlink,.rst-content h6 .pull-left.headerlink,.rst-content p .pull-left.headerlink,.rst-content table>caption .pull-left.headerlink,.rst-content tt.download span.pull-left:first-child,.wy-menu-vertical li.current>a button.pull-left.toctree-expand,.wy-menu-vertical li.on a button.pull-left.toctree-expand,.wy-menu-vertical li button.pull-left.toctree-expand{margin-right:.3em}.fa.pull-right,.pull-right.icon,.rst-content .code-block-caption .pull-right.headerlink,.rst-content .eqno .pull-right.headerlink,.rst-content .pull-right.admonition-title,.rst-content code.download span.pull-right:first-child,.rst-content dl dt .pull-right.headerlink,.rst-content h1 .pull-right.headerlink,.rst-content h2 .pull-right.headerlink,.rst-content h3 .pull-right.headerlink,.rst-content h4 .pull-right.headerlink,.rst-content h5 .pull-right.headerlink,.rst-content h6 .pull-right.headerlink,.rst-content p .pull-right.headerlink,.rst-content table>caption .pull-right.headerlink,.rst-content tt.download span.pull-right:first-child,.wy-menu-vertical li.current>a button.pull-right.toctree-expand,.wy-menu-vertical li.on a button.pull-right.toctree-expand,.wy-menu-vertical li button.pull-right.toctree-expand{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s linear infinite;animation:fa-spin 2s linear infinite}.fa-pulse{-webkit-animation:fa-spin 1s steps(8) infinite;animation:fa-spin 1s steps(8) infinite}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scaleX(-1);-ms-transform:scaleX(-1);transform:scaleX(-1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scaleY(-1);-ms-transform:scaleY(-1);transform:scaleY(-1)}:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:""}.fa-music:before{content:""}.fa-search:before,.icon-search:before{content:""}.fa-envelope-o:before{content:""}.fa-heart:before{content:""}.fa-star:before{content:""}.fa-star-o:before{content:""}.fa-user:before{content:""}.fa-film:before{content:""}.fa-th-large:before{content:""}.fa-th:before{content:""}.fa-th-list:before{content:""}.fa-check:before{content:""}.fa-close:before,.fa-remove:before,.fa-times:before{content:""}.fa-search-plus:before{content:""}.fa-search-minus:before{content:""}.fa-power-off:before{content:""}.fa-signal:before{content:""}.fa-cog:before,.fa-gear:before{content:""}.fa-trash-o:before{content:""}.fa-home:before,.icon-home:before{content:""}.fa-file-o:before{content:""}.fa-clock-o:before{content:""}.fa-road:before{content:""}.fa-download:before,.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{content:""}.fa-arrow-circle-o-down:before{content:""}.fa-arrow-circle-o-up:before{content:""}.fa-inbox:before{content:""}.fa-play-circle-o:before{content:""}.fa-repeat:before,.fa-rotate-right:before{content:""}.fa-refresh:before{content:""}.fa-list-alt:before{content:""}.fa-lock:before{content:""}.fa-flag:before{content:""}.fa-headphones:before{content:""}.fa-volume-off:before{content:""}.fa-volume-down:before{content:""}.fa-volume-up:before{content:""}.fa-qrcode:before{content:""}.fa-barcode:before{content:""}.fa-tag:before{content:""}.fa-tags:before{content:""}.fa-book:before,.icon-book:before{content:""}.fa-bookmark:before{content:""}.fa-print:before{content:""}.fa-camera:before{content:""}.fa-font:before{content:""}.fa-bold:before{content:""}.fa-italic:before{content:""}.fa-text-height:before{content:""}.fa-text-width:before{content:""}.fa-align-left:before{content:""}.fa-align-center:before{content:""}.fa-align-right:before{content:""}.fa-align-justify:before{content:""}.fa-list:before{content:""}.fa-dedent:before,.fa-outdent:before{content:""}.fa-indent:before{content:""}.fa-video-camera:before{content:""}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:""}.fa-pencil:before{content:""}.fa-map-marker:before{content:""}.fa-adjust:before{content:""}.fa-tint:before{content:""}.fa-edit:before,.fa-pencil-square-o:before{content:""}.fa-share-square-o:before{content:""}.fa-check-square-o:before{content:""}.fa-arrows:before{content:""}.fa-step-backward:before{content:""}.fa-fast-backward:before{content:""}.fa-backward:before{content:""}.fa-play:before{content:""}.fa-pause:before{content:""}.fa-stop:before{content:""}.fa-forward:before{content:""}.fa-fast-forward:before{content:""}.fa-step-forward:before{content:""}.fa-eject:before{content:""}.fa-chevron-left:before{content:""}.fa-chevron-right:before{content:""}.fa-plus-circle:before{content:""}.fa-minus-circle:before{content:""}.fa-times-circle:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before{content:""}.fa-check-circle:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before{content:""}.fa-question-circle:before{content:""}.fa-info-circle:before{content:""}.fa-crosshairs:before{content:""}.fa-times-circle-o:before{content:""}.fa-check-circle-o:before{content:""}.fa-ban:before{content:""}.fa-arrow-left:before{content:""}.fa-arrow-right:before{content:""}.fa-arrow-up:before{content:""}.fa-arrow-down:before{content:""}.fa-mail-forward:before,.fa-share:before{content:""}.fa-expand:before{content:""}.fa-compress:before{content:""}.fa-plus:before{content:""}.fa-minus:before{content:""}.fa-asterisk:before{content:""}.fa-exclamation-circle:before,.rst-content .admonition-title:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before{content:""}.fa-gift:before{content:""}.fa-leaf:before{content:""}.fa-fire:before,.icon-fire:before{content:""}.fa-eye:before{content:""}.fa-eye-slash:before{content:""}.fa-exclamation-triangle:before,.fa-warning:before{content:""}.fa-plane:before{content:""}.fa-calendar:before{content:""}.fa-random:before{content:""}.fa-comment:before{content:""}.fa-magnet:before{content:""}.fa-chevron-up:before{content:""}.fa-chevron-down:before{content:""}.fa-retweet:before{content:""}.fa-shopping-cart:before{content:""}.fa-folder:before{content:""}.fa-folder-open:before{content:""}.fa-arrows-v:before{content:""}.fa-arrows-h:before{content:""}.fa-bar-chart-o:before,.fa-bar-chart:before{content:""}.fa-twitter-square:before{content:""}.fa-facebook-square:before{content:""}.fa-camera-retro:before{content:""}.fa-key:before{content:""}.fa-cogs:before,.fa-gears:before{content:""}.fa-comments:before{content:""}.fa-thumbs-o-up:before{content:""}.fa-thumbs-o-down:before{content:""}.fa-star-half:before{content:""}.fa-heart-o:before{content:""}.fa-sign-out:before{content:""}.fa-linkedin-square:before{content:""}.fa-thumb-tack:before{content:""}.fa-external-link:before{content:""}.fa-sign-in:before{content:""}.fa-trophy:before{content:""}.fa-github-square:before{content:""}.fa-upload:before{content:""}.fa-lemon-o:before{content:""}.fa-phone:before{content:""}.fa-square-o:before{content:""}.fa-bookmark-o:before{content:""}.fa-phone-square:before{content:""}.fa-twitter:before{content:""}.fa-facebook-f:before,.fa-facebook:before{content:""}.fa-github:before,.icon-github:before{content:""}.fa-unlock:before{content:""}.fa-credit-card:before{content:""}.fa-feed:before,.fa-rss:before{content:""}.fa-hdd-o:before{content:""}.fa-bullhorn:before{content:""}.fa-bell:before{content:""}.fa-certificate:before{content:""}.fa-hand-o-right:before{content:""}.fa-hand-o-left:before{content:""}.fa-hand-o-up:before{content:""}.fa-hand-o-down:before{content:""}.fa-arrow-circle-left:before,.icon-circle-arrow-left:before{content:""}.fa-arrow-circle-right:before,.icon-circle-arrow-right:before{content:""}.fa-arrow-circle-up:before{content:""}.fa-arrow-circle-down:before{content:""}.fa-globe:before{content:""}.fa-wrench:before{content:""}.fa-tasks:before{content:""}.fa-filter:before{content:""}.fa-briefcase:before{content:""}.fa-arrows-alt:before{content:""}.fa-group:before,.fa-users:before{content:""}.fa-chain:before,.fa-link:before,.icon-link:before{content:""}.fa-cloud:before{content:""}.fa-flask:before{content:""}.fa-cut:before,.fa-scissors:before{content:""}.fa-copy:before,.fa-files-o:before{content:""}.fa-paperclip:before{content:""}.fa-floppy-o:before,.fa-save:before{content:""}.fa-square:before{content:""}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:""}.fa-list-ul:before{content:""}.fa-list-ol:before{content:""}.fa-strikethrough:before{content:""}.fa-underline:before{content:""}.fa-table:before{content:""}.fa-magic:before{content:""}.fa-truck:before{content:""}.fa-pinterest:before{content:""}.fa-pinterest-square:before{content:""}.fa-google-plus-square:before{content:""}.fa-google-plus:before{content:""}.fa-money:before{content:""}.fa-caret-down:before,.icon-caret-down:before,.wy-dropdown .caret:before{content:""}.fa-caret-up:before{content:""}.fa-caret-left:before{content:""}.fa-caret-right:before{content:""}.fa-columns:before{content:""}.fa-sort:before,.fa-unsorted:before{content:""}.fa-sort-desc:before,.fa-sort-down:before{content:""}.fa-sort-asc:before,.fa-sort-up:before{content:""}.fa-envelope:before{content:""}.fa-linkedin:before{content:""}.fa-rotate-left:before,.fa-undo:before{content:""}.fa-gavel:before,.fa-legal:before{content:""}.fa-dashboard:before,.fa-tachometer:before{content:""}.fa-comment-o:before{content:""}.fa-comments-o:before{content:""}.fa-bolt:before,.fa-flash:before{content:""}.fa-sitemap:before{content:""}.fa-umbrella:before{content:""}.fa-clipboard:before,.fa-paste:before{content:""}.fa-lightbulb-o:before{content:""}.fa-exchange:before{content:""}.fa-cloud-download:before{content:""}.fa-cloud-upload:before{content:""}.fa-user-md:before{content:""}.fa-stethoscope:before{content:""}.fa-suitcase:before{content:""}.fa-bell-o:before{content:""}.fa-coffee:before{content:""}.fa-cutlery:before{content:""}.fa-file-text-o:before{content:""}.fa-building-o:before{content:""}.fa-hospital-o:before{content:""}.fa-ambulance:before{content:""}.fa-medkit:before{content:""}.fa-fighter-jet:before{content:""}.fa-beer:before{content:""}.fa-h-square:before{content:""}.fa-plus-square:before{content:""}.fa-angle-double-left:before{content:""}.fa-angle-double-right:before{content:""}.fa-angle-double-up:before{content:""}.fa-angle-double-down:before{content:""}.fa-angle-left:before{content:""}.fa-angle-right:before{content:""}.fa-angle-up:before{content:""}.fa-angle-down:before{content:""}.fa-desktop:before{content:""}.fa-laptop:before{content:""}.fa-tablet:before{content:""}.fa-mobile-phone:before,.fa-mobile:before{content:""}.fa-circle-o:before{content:""}.fa-quote-left:before{content:""}.fa-quote-right:before{content:""}.fa-spinner:before{content:""}.fa-circle:before{content:""}.fa-mail-reply:before,.fa-reply:before{content:""}.fa-github-alt:before{content:""}.fa-folder-o:before{content:""}.fa-folder-open-o:before{content:""}.fa-smile-o:before{content:""}.fa-frown-o:before{content:""}.fa-meh-o:before{content:""}.fa-gamepad:before{content:""}.fa-keyboard-o:before{content:""}.fa-flag-o:before{content:""}.fa-flag-checkered:before{content:""}.fa-terminal:before{content:""}.fa-code:before{content:""}.fa-mail-reply-all:before,.fa-reply-all:before{content:""}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:""}.fa-location-arrow:before{content:""}.fa-crop:before{content:""}.fa-code-fork:before{content:""}.fa-chain-broken:before,.fa-unlink:before{content:""}.fa-question:before{content:""}.fa-info:before{content:""}.fa-exclamation:before{content:""}.fa-superscript:before{content:""}.fa-subscript:before{content:""}.fa-eraser:before{content:""}.fa-puzzle-piece:before{content:""}.fa-microphone:before{content:""}.fa-microphone-slash:before{content:""}.fa-shield:before{content:""}.fa-calendar-o:before{content:""}.fa-fire-extinguisher:before{content:""}.fa-rocket:before{content:""}.fa-maxcdn:before{content:""}.fa-chevron-circle-left:before{content:""}.fa-chevron-circle-right:before{content:""}.fa-chevron-circle-up:before{content:""}.fa-chevron-circle-down:before{content:""}.fa-html5:before{content:""}.fa-css3:before{content:""}.fa-anchor:before{content:""}.fa-unlock-alt:before{content:""}.fa-bullseye:before{content:""}.fa-ellipsis-h:before{content:""}.fa-ellipsis-v:before{content:""}.fa-rss-square:before{content:""}.fa-play-circle:before{content:""}.fa-ticket:before{content:""}.fa-minus-square:before{content:""}.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before{content:""}.fa-level-up:before{content:""}.fa-level-down:before{content:""}.fa-check-square:before{content:""}.fa-pencil-square:before{content:""}.fa-external-link-square:before{content:""}.fa-share-square:before{content:""}.fa-compass:before{content:""}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:""}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:""}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:""}.fa-eur:before,.fa-euro:before{content:""}.fa-gbp:before{content:""}.fa-dollar:before,.fa-usd:before{content:""}.fa-inr:before,.fa-rupee:before{content:""}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:""}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:""}.fa-krw:before,.fa-won:before{content:""}.fa-bitcoin:before,.fa-btc:before{content:""}.fa-file:before{content:""}.fa-file-text:before{content:""}.fa-sort-alpha-asc:before{content:""}.fa-sort-alpha-desc:before{content:""}.fa-sort-amount-asc:before{content:""}.fa-sort-amount-desc:before{content:""}.fa-sort-numeric-asc:before{content:""}.fa-sort-numeric-desc:before{content:""}.fa-thumbs-up:before{content:""}.fa-thumbs-down:before{content:""}.fa-youtube-square:before{content:""}.fa-youtube:before{content:""}.fa-xing:before{content:""}.fa-xing-square:before{content:""}.fa-youtube-play:before{content:""}.fa-dropbox:before{content:""}.fa-stack-overflow:before{content:""}.fa-instagram:before{content:""}.fa-flickr:before{content:""}.fa-adn:before{content:""}.fa-bitbucket:before,.icon-bitbucket:before{content:""}.fa-bitbucket-square:before{content:""}.fa-tumblr:before{content:""}.fa-tumblr-square:before{content:""}.fa-long-arrow-down:before{content:""}.fa-long-arrow-up:before{content:""}.fa-long-arrow-left:before{content:""}.fa-long-arrow-right:before{content:""}.fa-apple:before{content:""}.fa-windows:before{content:""}.fa-android:before{content:""}.fa-linux:before{content:""}.fa-dribbble:before{content:""}.fa-skype:before{content:""}.fa-foursquare:before{content:""}.fa-trello:before{content:""}.fa-female:before{content:""}.fa-male:before{content:""}.fa-gittip:before,.fa-gratipay:before{content:""}.fa-sun-o:before{content:""}.fa-moon-o:before{content:""}.fa-archive:before{content:""}.fa-bug:before{content:""}.fa-vk:before{content:""}.fa-weibo:before{content:""}.fa-renren:before{content:""}.fa-pagelines:before{content:""}.fa-stack-exchange:before{content:""}.fa-arrow-circle-o-right:before{content:""}.fa-arrow-circle-o-left:before{content:""}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:""}.fa-dot-circle-o:before{content:""}.fa-wheelchair:before{content:""}.fa-vimeo-square:before{content:""}.fa-try:before,.fa-turkish-lira:before{content:""}.fa-plus-square-o:before,.wy-menu-vertical li button.toctree-expand:before{content:""}.fa-space-shuttle:before{content:""}.fa-slack:before{content:""}.fa-envelope-square:before{content:""}.fa-wordpress:before{content:""}.fa-openid:before{content:""}.fa-bank:before,.fa-institution:before,.fa-university:before{content:""}.fa-graduation-cap:before,.fa-mortar-board:before{content:""}.fa-yahoo:before{content:""}.fa-google:before{content:""}.fa-reddit:before{content:""}.fa-reddit-square:before{content:""}.fa-stumbleupon-circle:before{content:""}.fa-stumbleupon:before{content:""}.fa-delicious:before{content:""}.fa-digg:before{content:""}.fa-pied-piper-pp:before{content:""}.fa-pied-piper-alt:before{content:""}.fa-drupal:before{content:""}.fa-joomla:before{content:""}.fa-language:before{content:""}.fa-fax:before{content:""}.fa-building:before{content:""}.fa-child:before{content:""}.fa-paw:before{content:""}.fa-spoon:before{content:""}.fa-cube:before{content:""}.fa-cubes:before{content:""}.fa-behance:before{content:""}.fa-behance-square:before{content:""}.fa-steam:before{content:""}.fa-steam-square:before{content:""}.fa-recycle:before{content:""}.fa-automobile:before,.fa-car:before{content:""}.fa-cab:before,.fa-taxi:before{content:""}.fa-tree:before{content:""}.fa-spotify:before{content:""}.fa-deviantart:before{content:""}.fa-soundcloud:before{content:""}.fa-database:before{content:""}.fa-file-pdf-o:before{content:""}.fa-file-word-o:before{content:""}.fa-file-excel-o:before{content:""}.fa-file-powerpoint-o:before{content:""}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:""}.fa-file-archive-o:before,.fa-file-zip-o:before{content:""}.fa-file-audio-o:before,.fa-file-sound-o:before{content:""}.fa-file-movie-o:before,.fa-file-video-o:before{content:""}.fa-file-code-o:before{content:""}.fa-vine:before{content:""}.fa-codepen:before{content:""}.fa-jsfiddle:before{content:""}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:""}.fa-circle-o-notch:before{content:""}.fa-ra:before,.fa-rebel:before,.fa-resistance:before{content:""}.fa-empire:before,.fa-ge:before{content:""}.fa-git-square:before{content:""}.fa-git:before{content:""}.fa-hacker-news:before,.fa-y-combinator-square:before,.fa-yc-square:before{content:""}.fa-tencent-weibo:before{content:""}.fa-qq:before{content:""}.fa-wechat:before,.fa-weixin:before{content:""}.fa-paper-plane:before,.fa-send:before{content:""}.fa-paper-plane-o:before,.fa-send-o:before{content:""}.fa-history:before{content:""}.fa-circle-thin:before{content:""}.fa-header:before{content:""}.fa-paragraph:before{content:""}.fa-sliders:before{content:""}.fa-share-alt:before{content:""}.fa-share-alt-square:before{content:""}.fa-bomb:before{content:""}.fa-futbol-o:before,.fa-soccer-ball-o:before{content:""}.fa-tty:before{content:""}.fa-binoculars:before{content:""}.fa-plug:before{content:""}.fa-slideshare:before{content:""}.fa-twitch:before{content:""}.fa-yelp:before{content:""}.fa-newspaper-o:before{content:""}.fa-wifi:before{content:""}.fa-calculator:before{content:""}.fa-paypal:before{content:""}.fa-google-wallet:before{content:""}.fa-cc-visa:before{content:""}.fa-cc-mastercard:before{content:""}.fa-cc-discover:before{content:""}.fa-cc-amex:before{content:""}.fa-cc-paypal:before{content:""}.fa-cc-stripe:before{content:""}.fa-bell-slash:before{content:""}.fa-bell-slash-o:before{content:""}.fa-trash:before{content:""}.fa-copyright:before{content:""}.fa-at:before{content:""}.fa-eyedropper:before{content:""}.fa-paint-brush:before{content:""}.fa-birthday-cake:before{content:""}.fa-area-chart:before{content:""}.fa-pie-chart:before{content:""}.fa-line-chart:before{content:""}.fa-lastfm:before{content:""}.fa-lastfm-square:before{content:""}.fa-toggle-off:before{content:""}.fa-toggle-on:before{content:""}.fa-bicycle:before{content:""}.fa-bus:before{content:""}.fa-ioxhost:before{content:""}.fa-angellist:before{content:""}.fa-cc:before{content:""}.fa-ils:before,.fa-shekel:before,.fa-sheqel:before{content:""}.fa-meanpath:before{content:""}.fa-buysellads:before{content:""}.fa-connectdevelop:before{content:""}.fa-dashcube:before{content:""}.fa-forumbee:before{content:""}.fa-leanpub:before{content:""}.fa-sellsy:before{content:""}.fa-shirtsinbulk:before{content:""}.fa-simplybuilt:before{content:""}.fa-skyatlas:before{content:""}.fa-cart-plus:before{content:""}.fa-cart-arrow-down:before{content:""}.fa-diamond:before{content:""}.fa-ship:before{content:""}.fa-user-secret:before{content:""}.fa-motorcycle:before{content:""}.fa-street-view:before{content:""}.fa-heartbeat:before{content:""}.fa-venus:before{content:""}.fa-mars:before{content:""}.fa-mercury:before{content:""}.fa-intersex:before,.fa-transgender:before{content:""}.fa-transgender-alt:before{content:""}.fa-venus-double:before{content:""}.fa-mars-double:before{content:""}.fa-venus-mars:before{content:""}.fa-mars-stroke:before{content:""}.fa-mars-stroke-v:before{content:""}.fa-mars-stroke-h:before{content:""}.fa-neuter:before{content:""}.fa-genderless:before{content:""}.fa-facebook-official:before{content:""}.fa-pinterest-p:before{content:""}.fa-whatsapp:before{content:""}.fa-server:before{content:""}.fa-user-plus:before{content:""}.fa-user-times:before{content:""}.fa-bed:before,.fa-hotel:before{content:""}.fa-viacoin:before{content:""}.fa-train:before{content:""}.fa-subway:before{content:""}.fa-medium:before{content:""}.fa-y-combinator:before,.fa-yc:before{content:""}.fa-optin-monster:before{content:""}.fa-opencart:before{content:""}.fa-expeditedssl:before{content:""}.fa-battery-4:before,.fa-battery-full:before,.fa-battery:before{content:""}.fa-battery-3:before,.fa-battery-three-quarters:before{content:""}.fa-battery-2:before,.fa-battery-half:before{content:""}.fa-battery-1:before,.fa-battery-quarter:before{content:""}.fa-battery-0:before,.fa-battery-empty:before{content:""}.fa-mouse-pointer:before{content:""}.fa-i-cursor:before{content:""}.fa-object-group:before{content:""}.fa-object-ungroup:before{content:""}.fa-sticky-note:before{content:""}.fa-sticky-note-o:before{content:""}.fa-cc-jcb:before{content:""}.fa-cc-diners-club:before{content:""}.fa-clone:before{content:""}.fa-balance-scale:before{content:""}.fa-hourglass-o:before{content:""}.fa-hourglass-1:before,.fa-hourglass-start:before{content:""}.fa-hourglass-2:before,.fa-hourglass-half:before{content:""}.fa-hourglass-3:before,.fa-hourglass-end:before{content:""}.fa-hourglass:before{content:""}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:""}.fa-hand-paper-o:before,.fa-hand-stop-o:before{content:""}.fa-hand-scissors-o:before{content:""}.fa-hand-lizard-o:before{content:""}.fa-hand-spock-o:before{content:""}.fa-hand-pointer-o:before{content:""}.fa-hand-peace-o:before{content:""}.fa-trademark:before{content:""}.fa-registered:before{content:""}.fa-creative-commons:before{content:""}.fa-gg:before{content:""}.fa-gg-circle:before{content:""}.fa-tripadvisor:before{content:""}.fa-odnoklassniki:before{content:""}.fa-odnoklassniki-square:before{content:""}.fa-get-pocket:before{content:""}.fa-wikipedia-w:before{content:""}.fa-safari:before{content:""}.fa-chrome:before{content:""}.fa-firefox:before{content:""}.fa-opera:before{content:""}.fa-internet-explorer:before{content:""}.fa-television:before,.fa-tv:before{content:""}.fa-contao:before{content:""}.fa-500px:before{content:""}.fa-amazon:before{content:""}.fa-calendar-plus-o:before{content:""}.fa-calendar-minus-o:before{content:""}.fa-calendar-times-o:before{content:""}.fa-calendar-check-o:before{content:""}.fa-industry:before{content:""}.fa-map-pin:before{content:""}.fa-map-signs:before{content:""}.fa-map-o:before{content:""}.fa-map:before{content:""}.fa-commenting:before{content:""}.fa-commenting-o:before{content:""}.fa-houzz:before{content:""}.fa-vimeo:before{content:""}.fa-black-tie:before{content:""}.fa-fonticons:before{content:""}.fa-reddit-alien:before{content:""}.fa-edge:before{content:""}.fa-credit-card-alt:before{content:""}.fa-codiepie:before{content:""}.fa-modx:before{content:""}.fa-fort-awesome:before{content:""}.fa-usb:before{content:""}.fa-product-hunt:before{content:""}.fa-mixcloud:before{content:""}.fa-scribd:before{content:""}.fa-pause-circle:before{content:""}.fa-pause-circle-o:before{content:""}.fa-stop-circle:before{content:""}.fa-stop-circle-o:before{content:""}.fa-shopping-bag:before{content:""}.fa-shopping-basket:before{content:""}.fa-hashtag:before{content:""}.fa-bluetooth:before{content:""}.fa-bluetooth-b:before{content:""}.fa-percent:before{content:""}.fa-gitlab:before,.icon-gitlab:before{content:""}.fa-wpbeginner:before{content:""}.fa-wpforms:before{content:""}.fa-envira:before{content:""}.fa-universal-access:before{content:""}.fa-wheelchair-alt:before{content:""}.fa-question-circle-o:before{content:""}.fa-blind:before{content:""}.fa-audio-description:before{content:""}.fa-volume-control-phone:before{content:""}.fa-braille:before{content:""}.fa-assistive-listening-systems:before{content:""}.fa-american-sign-language-interpreting:before,.fa-asl-interpreting:before{content:""}.fa-deaf:before,.fa-deafness:before,.fa-hard-of-hearing:before{content:""}.fa-glide:before{content:""}.fa-glide-g:before{content:""}.fa-sign-language:before,.fa-signing:before{content:""}.fa-low-vision:before{content:""}.fa-viadeo:before{content:""}.fa-viadeo-square:before{content:""}.fa-snapchat:before{content:""}.fa-snapchat-ghost:before{content:""}.fa-snapchat-square:before{content:""}.fa-pied-piper:before{content:""}.fa-first-order:before{content:""}.fa-yoast:before{content:""}.fa-themeisle:before{content:""}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:""}.fa-fa:before,.fa-font-awesome:before{content:""}.fa-handshake-o:before{content:""}.fa-envelope-open:before{content:""}.fa-envelope-open-o:before{content:""}.fa-linode:before{content:""}.fa-address-book:before{content:""}.fa-address-book-o:before{content:""}.fa-address-card:before,.fa-vcard:before{content:""}.fa-address-card-o:before,.fa-vcard-o:before{content:""}.fa-user-circle:before{content:""}.fa-user-circle-o:before{content:""}.fa-user-o:before{content:""}.fa-id-badge:before{content:""}.fa-drivers-license:before,.fa-id-card:before{content:""}.fa-drivers-license-o:before,.fa-id-card-o:before{content:""}.fa-quora:before{content:""}.fa-free-code-camp:before{content:""}.fa-telegram:before{content:""}.fa-thermometer-4:before,.fa-thermometer-full:before,.fa-thermometer:before{content:""}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:""}.fa-thermometer-2:before,.fa-thermometer-half:before{content:""}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:""}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:""}.fa-shower:before{content:""}.fa-bath:before,.fa-bathtub:before,.fa-s15:before{content:""}.fa-podcast:before{content:""}.fa-window-maximize:before{content:""}.fa-window-minimize:before{content:""}.fa-window-restore:before{content:""}.fa-times-rectangle:before,.fa-window-close:before{content:""}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:""}.fa-bandcamp:before{content:""}.fa-grav:before{content:""}.fa-etsy:before{content:""}.fa-imdb:before{content:""}.fa-ravelry:before{content:""}.fa-eercast:before{content:""}.fa-microchip:before{content:""}.fa-snowflake-o:before{content:""}.fa-superpowers:before{content:""}.fa-wpexplorer:before{content:""}.fa-meetup:before{content:""}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.fa,.icon,.rst-content .admonition-title,.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content code.download span:first-child,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink,.rst-content tt.download span:first-child,.wy-dropdown .caret,.wy-inline-validate.wy-inline-validate-danger .wy-input-context,.wy-inline-validate.wy-inline-validate-info .wy-input-context,.wy-inline-validate.wy-inline-validate-success .wy-input-context,.wy-inline-validate.wy-inline-validate-warning .wy-input-context,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li button.toctree-expand{font-family:inherit}.fa:before,.icon:before,.rst-content .admonition-title:before,.rst-content .code-block-caption .headerlink:before,.rst-content .eqno .headerlink:before,.rst-content code.download span:first-child:before,.rst-content dl dt .headerlink:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content p .headerlink:before,.rst-content table>caption .headerlink:before,.rst-content tt.download span:first-child:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before,.wy-menu-vertical li button.toctree-expand:before{font-family:FontAwesome;display:inline-block;font-style:normal;font-weight:400;line-height:1;text-decoration:inherit}.rst-content .code-block-caption a .headerlink,.rst-content .eqno a .headerlink,.rst-content a .admonition-title,.rst-content code.download a span:first-child,.rst-content dl dt a .headerlink,.rst-content h1 a .headerlink,.rst-content h2 a .headerlink,.rst-content h3 a .headerlink,.rst-content h4 a .headerlink,.rst-content h5 a .headerlink,.rst-content h6 a .headerlink,.rst-content p.caption a .headerlink,.rst-content p a .headerlink,.rst-content table>caption a .headerlink,.rst-content tt.download a span:first-child,.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand,.wy-menu-vertical li a button.toctree-expand,a .fa,a .icon,a .rst-content .admonition-title,a .rst-content .code-block-caption .headerlink,a .rst-content .eqno .headerlink,a .rst-content code.download span:first-child,a .rst-content dl dt .headerlink,a .rst-content h1 .headerlink,a .rst-content h2 .headerlink,a .rst-content h3 .headerlink,a .rst-content h4 .headerlink,a .rst-content h5 .headerlink,a .rst-content h6 .headerlink,a .rst-content p.caption .headerlink,a .rst-content p .headerlink,a .rst-content table>caption .headerlink,a .rst-content tt.download span:first-child,a .wy-menu-vertical li button.toctree-expand{display:inline-block;text-decoration:inherit}.btn .fa,.btn .icon,.btn .rst-content .admonition-title,.btn .rst-content .code-block-caption .headerlink,.btn .rst-content .eqno .headerlink,.btn .rst-content code.download span:first-child,.btn .rst-content dl dt .headerlink,.btn .rst-content h1 .headerlink,.btn .rst-content h2 .headerlink,.btn .rst-content h3 .headerlink,.btn .rst-content h4 .headerlink,.btn .rst-content h5 .headerlink,.btn .rst-content h6 .headerlink,.btn .rst-content p .headerlink,.btn .rst-content table>caption .headerlink,.btn .rst-content tt.download span:first-child,.btn .wy-menu-vertical li.current>a button.toctree-expand,.btn .wy-menu-vertical li.on a button.toctree-expand,.btn .wy-menu-vertical li button.toctree-expand,.nav .fa,.nav .icon,.nav .rst-content .admonition-title,.nav .rst-content .code-block-caption .headerlink,.nav .rst-content .eqno .headerlink,.nav .rst-content code.download span:first-child,.nav .rst-content dl dt .headerlink,.nav .rst-content h1 .headerlink,.nav .rst-content h2 .headerlink,.nav .rst-content h3 .headerlink,.nav .rst-content h4 .headerlink,.nav .rst-content h5 .headerlink,.nav .rst-content h6 .headerlink,.nav .rst-content p .headerlink,.nav .rst-content table>caption .headerlink,.nav .rst-content tt.download span:first-child,.nav .wy-menu-vertical li.current>a button.toctree-expand,.nav .wy-menu-vertical li.on a button.toctree-expand,.nav .wy-menu-vertical li button.toctree-expand,.rst-content .btn .admonition-title,.rst-content .code-block-caption .btn .headerlink,.rst-content .code-block-caption .nav .headerlink,.rst-content .eqno .btn .headerlink,.rst-content .eqno .nav .headerlink,.rst-content .nav .admonition-title,.rst-content code.download .btn span:first-child,.rst-content code.download .nav span:first-child,.rst-content dl dt .btn .headerlink,.rst-content dl dt .nav .headerlink,.rst-content h1 .btn .headerlink,.rst-content h1 .nav .headerlink,.rst-content h2 .btn .headerlink,.rst-content h2 .nav .headerlink,.rst-content h3 .btn .headerlink,.rst-content h3 .nav .headerlink,.rst-content h4 .btn .headerlink,.rst-content h4 .nav .headerlink,.rst-content h5 .btn .headerlink,.rst-content h5 .nav .headerlink,.rst-content h6 .btn .headerlink,.rst-content h6 .nav .headerlink,.rst-content p .btn .headerlink,.rst-content p .nav .headerlink,.rst-content table>caption .btn .headerlink,.rst-content table>caption .nav .headerlink,.rst-content tt.download .btn span:first-child,.rst-content tt.download .nav span:first-child,.wy-menu-vertical li .btn button.toctree-expand,.wy-menu-vertical li.current>a .btn button.toctree-expand,.wy-menu-vertical li.current>a .nav button.toctree-expand,.wy-menu-vertical li .nav button.toctree-expand,.wy-menu-vertical li.on a .btn button.toctree-expand,.wy-menu-vertical li.on a .nav button.toctree-expand{display:inline}.btn .fa-large.icon,.btn .fa.fa-large,.btn .rst-content .code-block-caption .fa-large.headerlink,.btn .rst-content .eqno .fa-large.headerlink,.btn .rst-content .fa-large.admonition-title,.btn .rst-content code.download span.fa-large:first-child,.btn .rst-content dl dt .fa-large.headerlink,.btn .rst-content h1 .fa-large.headerlink,.btn .rst-content h2 .fa-large.headerlink,.btn .rst-content h3 .fa-large.headerlink,.btn .rst-content h4 .fa-large.headerlink,.btn .rst-content h5 .fa-large.headerlink,.btn .rst-content h6 .fa-large.headerlink,.btn .rst-content p .fa-large.headerlink,.btn .rst-content table>caption .fa-large.headerlink,.btn .rst-content tt.download span.fa-large:first-child,.btn .wy-menu-vertical li button.fa-large.toctree-expand,.nav .fa-large.icon,.nav .fa.fa-large,.nav .rst-content .code-block-caption .fa-large.headerlink,.nav .rst-content .eqno .fa-large.headerlink,.nav .rst-content .fa-large.admonition-title,.nav .rst-content code.download span.fa-large:first-child,.nav .rst-content dl dt .fa-large.headerlink,.nav .rst-content h1 .fa-large.headerlink,.nav .rst-content h2 .fa-large.headerlink,.nav .rst-content h3 .fa-large.headerlink,.nav .rst-content h4 .fa-large.headerlink,.nav .rst-content h5 .fa-large.headerlink,.nav .rst-content h6 .fa-large.headerlink,.nav .rst-content p .fa-large.headerlink,.nav .rst-content table>caption .fa-large.headerlink,.nav .rst-content tt.download span.fa-large:first-child,.nav .wy-menu-vertical li button.fa-large.toctree-expand,.rst-content .btn .fa-large.admonition-title,.rst-content .code-block-caption .btn .fa-large.headerlink,.rst-content .code-block-caption .nav .fa-large.headerlink,.rst-content .eqno .btn .fa-large.headerlink,.rst-content .eqno .nav .fa-large.headerlink,.rst-content .nav .fa-large.admonition-title,.rst-content code.download .btn span.fa-large:first-child,.rst-content code.download .nav span.fa-large:first-child,.rst-content dl dt .btn .fa-large.headerlink,.rst-content dl dt .nav .fa-large.headerlink,.rst-content h1 .btn .fa-large.headerlink,.rst-content h1 .nav .fa-large.headerlink,.rst-content h2 .btn .fa-large.headerlink,.rst-content h2 .nav .fa-large.headerlink,.rst-content h3 .btn .fa-large.headerlink,.rst-content h3 .nav .fa-large.headerlink,.rst-content h4 .btn .fa-large.headerlink,.rst-content h4 .nav .fa-large.headerlink,.rst-content h5 .btn .fa-large.headerlink,.rst-content h5 .nav .fa-large.headerlink,.rst-content h6 .btn .fa-large.headerlink,.rst-content h6 .nav .fa-large.headerlink,.rst-content p .btn .fa-large.headerlink,.rst-content p .nav .fa-large.headerlink,.rst-content table>caption .btn .fa-large.headerlink,.rst-content table>caption .nav .fa-large.headerlink,.rst-content tt.download .btn span.fa-large:first-child,.rst-content tt.download .nav span.fa-large:first-child,.wy-menu-vertical li .btn button.fa-large.toctree-expand,.wy-menu-vertical li .nav button.fa-large.toctree-expand{line-height:.9em}.btn .fa-spin.icon,.btn .fa.fa-spin,.btn .rst-content .code-block-caption .fa-spin.headerlink,.btn .rst-content .eqno .fa-spin.headerlink,.btn .rst-content .fa-spin.admonition-title,.btn .rst-content code.download span.fa-spin:first-child,.btn .rst-content dl dt .fa-spin.headerlink,.btn .rst-content h1 .fa-spin.headerlink,.btn .rst-content h2 .fa-spin.headerlink,.btn .rst-content h3 .fa-spin.headerlink,.btn .rst-content h4 .fa-spin.headerlink,.btn .rst-content h5 .fa-spin.headerlink,.btn .rst-content h6 .fa-spin.headerlink,.btn .rst-content p .fa-spin.headerlink,.btn .rst-content table>caption .fa-spin.headerlink,.btn .rst-content tt.download span.fa-spin:first-child,.btn .wy-menu-vertical li button.fa-spin.toctree-expand,.nav .fa-spin.icon,.nav .fa.fa-spin,.nav .rst-content .code-block-caption .fa-spin.headerlink,.nav .rst-content .eqno .fa-spin.headerlink,.nav .rst-content .fa-spin.admonition-title,.nav .rst-content code.download span.fa-spin:first-child,.nav .rst-content dl dt .fa-spin.headerlink,.nav .rst-content h1 .fa-spin.headerlink,.nav .rst-content h2 .fa-spin.headerlink,.nav .rst-content h3 .fa-spin.headerlink,.nav .rst-content h4 .fa-spin.headerlink,.nav .rst-content h5 .fa-spin.headerlink,.nav .rst-content h6 .fa-spin.headerlink,.nav .rst-content p .fa-spin.headerlink,.nav .rst-content table>caption .fa-spin.headerlink,.nav .rst-content tt.download span.fa-spin:first-child,.nav .wy-menu-vertical li button.fa-spin.toctree-expand,.rst-content .btn .fa-spin.admonition-title,.rst-content .code-block-caption .btn .fa-spin.headerlink,.rst-content .code-block-caption .nav .fa-spin.headerlink,.rst-content .eqno .btn .fa-spin.headerlink,.rst-content .eqno .nav .fa-spin.headerlink,.rst-content .nav .fa-spin.admonition-title,.rst-content code.download .btn span.fa-spin:first-child,.rst-content code.download .nav span.fa-spin:first-child,.rst-content dl dt .btn .fa-spin.headerlink,.rst-content dl dt .nav .fa-spin.headerlink,.rst-content h1 .btn .fa-spin.headerlink,.rst-content h1 .nav .fa-spin.headerlink,.rst-content h2 .btn .fa-spin.headerlink,.rst-content h2 .nav .fa-spin.headerlink,.rst-content h3 .btn .fa-spin.headerlink,.rst-content h3 .nav .fa-spin.headerlink,.rst-content h4 .btn .fa-spin.headerlink,.rst-content h4 .nav .fa-spin.headerlink,.rst-content h5 .btn .fa-spin.headerlink,.rst-content h5 .nav .fa-spin.headerlink,.rst-content h6 .btn .fa-spin.headerlink,.rst-content h6 .nav .fa-spin.headerlink,.rst-content p .btn .fa-spin.headerlink,.rst-content p .nav .fa-spin.headerlink,.rst-content table>caption .btn .fa-spin.headerlink,.rst-content table>caption .nav .fa-spin.headerlink,.rst-content tt.download .btn span.fa-spin:first-child,.rst-content tt.download .nav span.fa-spin:first-child,.wy-menu-vertical li .btn button.fa-spin.toctree-expand,.wy-menu-vertical li .nav button.fa-spin.toctree-expand{display:inline-block}.btn.fa:before,.btn.icon:before,.rst-content .btn.admonition-title:before,.rst-content .code-block-caption .btn.headerlink:before,.rst-content .eqno .btn.headerlink:before,.rst-content code.download span.btn:first-child:before,.rst-content dl dt .btn.headerlink:before,.rst-content h1 .btn.headerlink:before,.rst-content h2 .btn.headerlink:before,.rst-content h3 .btn.headerlink:before,.rst-content h4 .btn.headerlink:before,.rst-content h5 .btn.headerlink:before,.rst-content h6 .btn.headerlink:before,.rst-content p .btn.headerlink:before,.rst-content table>caption .btn.headerlink:before,.rst-content tt.download span.btn:first-child:before,.wy-menu-vertical li button.btn.toctree-expand:before{opacity:.5;-webkit-transition:opacity .05s ease-in;-moz-transition:opacity .05s ease-in;transition:opacity .05s ease-in}.btn.fa:hover:before,.btn.icon:hover:before,.rst-content .btn.admonition-title:hover:before,.rst-content .code-block-caption .btn.headerlink:hover:before,.rst-content .eqno .btn.headerlink:hover:before,.rst-content code.download span.btn:first-child:hover:before,.rst-content dl dt .btn.headerlink:hover:before,.rst-content h1 .btn.headerlink:hover:before,.rst-content h2 .btn.headerlink:hover:before,.rst-content h3 .btn.headerlink:hover:before,.rst-content h4 .btn.headerlink:hover:before,.rst-content h5 .btn.headerlink:hover:before,.rst-content h6 .btn.headerlink:hover:before,.rst-content p .btn.headerlink:hover:before,.rst-content table>caption .btn.headerlink:hover:before,.rst-content tt.download span.btn:first-child:hover:before,.wy-menu-vertical li button.btn.toctree-expand:hover:before{opacity:1}.btn-mini .fa:before,.btn-mini .icon:before,.btn-mini .rst-content .admonition-title:before,.btn-mini .rst-content .code-block-caption .headerlink:before,.btn-mini .rst-content .eqno .headerlink:before,.btn-mini .rst-content code.download span:first-child:before,.btn-mini .rst-content dl dt .headerlink:before,.btn-mini .rst-content h1 .headerlink:before,.btn-mini .rst-content h2 .headerlink:before,.btn-mini .rst-content h3 .headerlink:before,.btn-mini .rst-content h4 .headerlink:before,.btn-mini .rst-content h5 .headerlink:before,.btn-mini .rst-content h6 .headerlink:before,.btn-mini .rst-content p .headerlink:before,.btn-mini .rst-content table>caption .headerlink:before,.btn-mini .rst-content tt.download span:first-child:before,.btn-mini .wy-menu-vertical li button.toctree-expand:before,.rst-content .btn-mini .admonition-title:before,.rst-content .code-block-caption .btn-mini .headerlink:before,.rst-content .eqno .btn-mini .headerlink:before,.rst-content code.download .btn-mini span:first-child:before,.rst-content dl dt .btn-mini .headerlink:before,.rst-content h1 .btn-mini .headerlink:before,.rst-content h2 .btn-mini .headerlink:before,.rst-content h3 .btn-mini .headerlink:before,.rst-content h4 .btn-mini .headerlink:before,.rst-content h5 .btn-mini .headerlink:before,.rst-content h6 .btn-mini .headerlink:before,.rst-content p .btn-mini .headerlink:before,.rst-content table>caption .btn-mini .headerlink:before,.rst-content tt.download .btn-mini span:first-child:before,.wy-menu-vertical li .btn-mini button.toctree-expand:before{font-size:14px;vertical-align:-15%}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning,.wy-alert{padding:12px;line-height:24px;margin-bottom:24px;background:#e7f2fa}.rst-content .admonition-title,.wy-alert-title{font-weight:700;display:block;color:#fff;background:#6ab0de;padding:6px 12px;margin:-12px -12px 12px}.rst-content .danger,.rst-content .error,.rst-content .wy-alert-danger.admonition,.rst-content .wy-alert-danger.admonition-todo,.rst-content .wy-alert-danger.attention,.rst-content .wy-alert-danger.caution,.rst-content .wy-alert-danger.hint,.rst-content .wy-alert-danger.important,.rst-content .wy-alert-danger.note,.rst-content .wy-alert-danger.seealso,.rst-content .wy-alert-danger.tip,.rst-content .wy-alert-danger.warning,.wy-alert.wy-alert-danger{background:#fdf3f2}.rst-content .danger .admonition-title,.rst-content .danger .wy-alert-title,.rst-content .error .admonition-title,.rst-content .error .wy-alert-title,.rst-content .wy-alert-danger.admonition-todo .admonition-title,.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,.rst-content .wy-alert-danger.admonition .admonition-title,.rst-content .wy-alert-danger.admonition .wy-alert-title,.rst-content .wy-alert-danger.attention .admonition-title,.rst-content .wy-alert-danger.attention .wy-alert-title,.rst-content .wy-alert-danger.caution .admonition-title,.rst-content .wy-alert-danger.caution .wy-alert-title,.rst-content .wy-alert-danger.hint .admonition-title,.rst-content .wy-alert-danger.hint .wy-alert-title,.rst-content .wy-alert-danger.important .admonition-title,.rst-content .wy-alert-danger.important .wy-alert-title,.rst-content .wy-alert-danger.note .admonition-title,.rst-content .wy-alert-danger.note .wy-alert-title,.rst-content .wy-alert-danger.seealso .admonition-title,.rst-content .wy-alert-danger.seealso .wy-alert-title,.rst-content .wy-alert-danger.tip .admonition-title,.rst-content .wy-alert-danger.tip .wy-alert-title,.rst-content .wy-alert-danger.warning .admonition-title,.rst-content .wy-alert-danger.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-danger .admonition-title,.wy-alert.wy-alert-danger .rst-content .admonition-title,.wy-alert.wy-alert-danger .wy-alert-title{background:#f29f97}.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .warning,.rst-content .wy-alert-warning.admonition,.rst-content .wy-alert-warning.danger,.rst-content .wy-alert-warning.error,.rst-content .wy-alert-warning.hint,.rst-content .wy-alert-warning.important,.rst-content .wy-alert-warning.note,.rst-content .wy-alert-warning.seealso,.rst-content .wy-alert-warning.tip,.wy-alert.wy-alert-warning{background:#ffedcc}.rst-content .admonition-todo .admonition-title,.rst-content .admonition-todo .wy-alert-title,.rst-content .attention .admonition-title,.rst-content .attention .wy-alert-title,.rst-content .caution .admonition-title,.rst-content .caution .wy-alert-title,.rst-content .warning .admonition-title,.rst-content .warning .wy-alert-title,.rst-content .wy-alert-warning.admonition .admonition-title,.rst-content .wy-alert-warning.admonition .wy-alert-title,.rst-content .wy-alert-warning.danger .admonition-title,.rst-content .wy-alert-warning.danger .wy-alert-title,.rst-content .wy-alert-warning.error .admonition-title,.rst-content .wy-alert-warning.error .wy-alert-title,.rst-content .wy-alert-warning.hint .admonition-title,.rst-content .wy-alert-warning.hint .wy-alert-title,.rst-content .wy-alert-warning.important .admonition-title,.rst-content .wy-alert-warning.important .wy-alert-title,.rst-content .wy-alert-warning.note .admonition-title,.rst-content .wy-alert-warning.note .wy-alert-title,.rst-content .wy-alert-warning.seealso .admonition-title,.rst-content .wy-alert-warning.seealso .wy-alert-title,.rst-content .wy-alert-warning.tip .admonition-title,.rst-content .wy-alert-warning.tip .wy-alert-title,.rst-content .wy-alert.wy-alert-warning .admonition-title,.wy-alert.wy-alert-warning .rst-content .admonition-title,.wy-alert.wy-alert-warning .wy-alert-title{background:#f0b37e}.rst-content .note,.rst-content .seealso,.rst-content .wy-alert-info.admonition,.rst-content .wy-alert-info.admonition-todo,.rst-content .wy-alert-info.attention,.rst-content .wy-alert-info.caution,.rst-content .wy-alert-info.danger,.rst-content .wy-alert-info.error,.rst-content .wy-alert-info.hint,.rst-content .wy-alert-info.important,.rst-content .wy-alert-info.tip,.rst-content .wy-alert-info.warning,.wy-alert.wy-alert-info{background:#e7f2fa}.rst-content .note .admonition-title,.rst-content .note .wy-alert-title,.rst-content .seealso .admonition-title,.rst-content .seealso .wy-alert-title,.rst-content .wy-alert-info.admonition-todo .admonition-title,.rst-content .wy-alert-info.admonition-todo .wy-alert-title,.rst-content .wy-alert-info.admonition .admonition-title,.rst-content .wy-alert-info.admonition .wy-alert-title,.rst-content .wy-alert-info.attention .admonition-title,.rst-content .wy-alert-info.attention .wy-alert-title,.rst-content .wy-alert-info.caution .admonition-title,.rst-content .wy-alert-info.caution .wy-alert-title,.rst-content .wy-alert-info.danger .admonition-title,.rst-content .wy-alert-info.danger .wy-alert-title,.rst-content .wy-alert-info.error .admonition-title,.rst-content .wy-alert-info.error .wy-alert-title,.rst-content .wy-alert-info.hint .admonition-title,.rst-content .wy-alert-info.hint .wy-alert-title,.rst-content .wy-alert-info.important .admonition-title,.rst-content .wy-alert-info.important .wy-alert-title,.rst-content .wy-alert-info.tip .admonition-title,.rst-content .wy-alert-info.tip .wy-alert-title,.rst-content .wy-alert-info.warning .admonition-title,.rst-content .wy-alert-info.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-info .admonition-title,.wy-alert.wy-alert-info .rst-content .admonition-title,.wy-alert.wy-alert-info .wy-alert-title{background:#6ab0de}.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .wy-alert-success.admonition,.rst-content .wy-alert-success.admonition-todo,.rst-content .wy-alert-success.attention,.rst-content .wy-alert-success.caution,.rst-content .wy-alert-success.danger,.rst-content .wy-alert-success.error,.rst-content .wy-alert-success.note,.rst-content .wy-alert-success.seealso,.rst-content .wy-alert-success.warning,.wy-alert.wy-alert-success{background:#dbfaf4}.rst-content .hint .admonition-title,.rst-content .hint .wy-alert-title,.rst-content .important .admonition-title,.rst-content .important .wy-alert-title,.rst-content .tip .admonition-title,.rst-content .tip .wy-alert-title,.rst-content .wy-alert-success.admonition-todo .admonition-title,.rst-content .wy-alert-success.admonition-todo .wy-alert-title,.rst-content .wy-alert-success.admonition .admonition-title,.rst-content .wy-alert-success.admonition .wy-alert-title,.rst-content .wy-alert-success.attention .admonition-title,.rst-content .wy-alert-success.attention .wy-alert-title,.rst-content .wy-alert-success.caution .admonition-title,.rst-content .wy-alert-success.caution .wy-alert-title,.rst-content .wy-alert-success.danger .admonition-title,.rst-content .wy-alert-success.danger .wy-alert-title,.rst-content .wy-alert-success.error .admonition-title,.rst-content .wy-alert-success.error .wy-alert-title,.rst-content .wy-alert-success.note .admonition-title,.rst-content .wy-alert-success.note .wy-alert-title,.rst-content .wy-alert-success.seealso .admonition-title,.rst-content .wy-alert-success.seealso .wy-alert-title,.rst-content .wy-alert-success.warning .admonition-title,.rst-content .wy-alert-success.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-success .admonition-title,.wy-alert.wy-alert-success .rst-content .admonition-title,.wy-alert.wy-alert-success .wy-alert-title{background:#1abc9c}.rst-content .wy-alert-neutral.admonition,.rst-content .wy-alert-neutral.admonition-todo,.rst-content .wy-alert-neutral.attention,.rst-content .wy-alert-neutral.caution,.rst-content .wy-alert-neutral.danger,.rst-content .wy-alert-neutral.error,.rst-content .wy-alert-neutral.hint,.rst-content .wy-alert-neutral.important,.rst-content .wy-alert-neutral.note,.rst-content .wy-alert-neutral.seealso,.rst-content .wy-alert-neutral.tip,.rst-content .wy-alert-neutral.warning,.wy-alert.wy-alert-neutral{background:#f3f6f6}.rst-content .wy-alert-neutral.admonition-todo .admonition-title,.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,.rst-content .wy-alert-neutral.admonition .admonition-title,.rst-content .wy-alert-neutral.admonition .wy-alert-title,.rst-content .wy-alert-neutral.attention .admonition-title,.rst-content .wy-alert-neutral.attention .wy-alert-title,.rst-content .wy-alert-neutral.caution .admonition-title,.rst-content .wy-alert-neutral.caution .wy-alert-title,.rst-content .wy-alert-neutral.danger .admonition-title,.rst-content .wy-alert-neutral.danger .wy-alert-title,.rst-content .wy-alert-neutral.error .admonition-title,.rst-content .wy-alert-neutral.error .wy-alert-title,.rst-content .wy-alert-neutral.hint .admonition-title,.rst-content .wy-alert-neutral.hint .wy-alert-title,.rst-content .wy-alert-neutral.important .admonition-title,.rst-content .wy-alert-neutral.important .wy-alert-title,.rst-content .wy-alert-neutral.note .admonition-title,.rst-content .wy-alert-neutral.note .wy-alert-title,.rst-content .wy-alert-neutral.seealso .admonition-title,.rst-content .wy-alert-neutral.seealso .wy-alert-title,.rst-content .wy-alert-neutral.tip .admonition-title,.rst-content .wy-alert-neutral.tip .wy-alert-title,.rst-content .wy-alert-neutral.warning .admonition-title,.rst-content .wy-alert-neutral.warning .wy-alert-title,.rst-content .wy-alert.wy-alert-neutral .admonition-title,.wy-alert.wy-alert-neutral .rst-content .admonition-title,.wy-alert.wy-alert-neutral .wy-alert-title{color:#404040;background:#e1e4e5}.rst-content .wy-alert-neutral.admonition-todo a,.rst-content .wy-alert-neutral.admonition a,.rst-content .wy-alert-neutral.attention a,.rst-content .wy-alert-neutral.caution a,.rst-content .wy-alert-neutral.danger a,.rst-content .wy-alert-neutral.error a,.rst-content .wy-alert-neutral.hint a,.rst-content .wy-alert-neutral.important a,.rst-content .wy-alert-neutral.note a,.rst-content .wy-alert-neutral.seealso a,.rst-content .wy-alert-neutral.tip a,.rst-content .wy-alert-neutral.warning a,.wy-alert.wy-alert-neutral a{color:#2980b9}.rst-content .admonition-todo p:last-child,.rst-content .admonition p:last-child,.rst-content .attention p:last-child,.rst-content .caution p:last-child,.rst-content .danger p:last-child,.rst-content .error p:last-child,.rst-content .hint p:last-child,.rst-content .important p:last-child,.rst-content .note p:last-child,.rst-content .seealso p:last-child,.rst-content .tip p:last-child,.rst-content .warning p:last-child,.wy-alert p:last-child{margin-bottom:0}.wy-tray-container{position:fixed;bottom:0;left:0;z-index:600}.wy-tray-container li{display:block;width:300px;background:transparent;color:#fff;text-align:center;box-shadow:0 5px 5px 0 rgba(0,0,0,.1);padding:0 24px;min-width:20%;opacity:0;height:0;line-height:56px;overflow:hidden;-webkit-transition:all .3s ease-in;-moz-transition:all .3s ease-in;transition:all .3s ease-in}.wy-tray-container li.wy-tray-item-success{background:#27ae60}.wy-tray-container li.wy-tray-item-info{background:#2980b9}.wy-tray-container li.wy-tray-item-warning{background:#e67e22}.wy-tray-container li.wy-tray-item-danger{background:#e74c3c}.wy-tray-container li.on{opacity:1;height:56px}@media screen and (max-width:768px){.wy-tray-container{bottom:auto;top:0;width:100%}.wy-tray-container li{width:100%}}button{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle;cursor:pointer;line-height:normal;-webkit-appearance:button;*overflow:visible}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}button[disabled]{cursor:default}.btn{display:inline-block;border-radius:2px;line-height:normal;white-space:nowrap;text-align:center;cursor:pointer;font-size:100%;padding:6px 12px 8px;color:#fff;border:1px solid rgba(0,0,0,.1);background-color:#27ae60;text-decoration:none;font-weight:400;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 2px -1px hsla(0,0%,100%,.5),inset 0 -2px 0 0 rgba(0,0,0,.1);outline-none:false;vertical-align:middle;*display:inline;zoom:1;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:all .1s linear;-moz-transition:all .1s linear;transition:all .1s linear}.btn-hover{background:#2e8ece;color:#fff}.btn:hover{background:#2cc36b;color:#fff}.btn:focus{background:#2cc36b;outline:0}.btn:active{box-shadow:inset 0 -1px 0 0 rgba(0,0,0,.05),inset 0 2px 0 0 rgba(0,0,0,.1);padding:8px 12px 6px}.btn:visited{color:#fff}.btn-disabled,.btn-disabled:active,.btn-disabled:focus,.btn-disabled:hover,.btn:disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:.4;cursor:not-allowed;box-shadow:none}.btn::-moz-focus-inner{padding:0;border:0}.btn-small{font-size:80%}.btn-info{background-color:#2980b9!important}.btn-info:hover{background-color:#2e8ece!important}.btn-neutral{background-color:#f3f6f6!important;color:#404040!important}.btn-neutral:hover{background-color:#e5ebeb!important;color:#404040}.btn-neutral:visited{color:#404040!important}.btn-success{background-color:#27ae60!important}.btn-success:hover{background-color:#295!important}.btn-danger{background-color:#e74c3c!important}.btn-danger:hover{background-color:#ea6153!important}.btn-warning{background-color:#e67e22!important}.btn-warning:hover{background-color:#e98b39!important}.btn-invert{background-color:#222}.btn-invert:hover{background-color:#2f2f2f!important}.btn-link{background-color:transparent!important;color:#2980b9;box-shadow:none;border-color:transparent!important}.btn-link:active,.btn-link:hover{background-color:transparent!important;color:#409ad5!important;box-shadow:none}.btn-link:visited{color:#9b59b6}.wy-btn-group .btn,.wy-control .btn{vertical-align:middle}.wy-btn-group{margin-bottom:24px;*zoom:1}.wy-btn-group:after,.wy-btn-group:before{display:table;content:""}.wy-btn-group:after{clear:both}.wy-dropdown{position:relative;display:inline-block}.wy-dropdown-active .wy-dropdown-menu{display:block}.wy-dropdown-menu{position:absolute;left:0;display:none;float:left;top:100%;min-width:100%;background:#fcfcfc;z-index:100;border:1px solid #cfd7dd;box-shadow:0 2px 2px 0 rgba(0,0,0,.1);padding:12px}.wy-dropdown-menu>dd>a{display:block;clear:both;color:#404040;white-space:nowrap;font-size:90%;padding:0 12px;cursor:pointer}.wy-dropdown-menu>dd>a:hover{background:#2980b9;color:#fff}.wy-dropdown-menu>dd.divider{border-top:1px solid #cfd7dd;margin:6px 0}.wy-dropdown-menu>dd.search{padding-bottom:12px}.wy-dropdown-menu>dd.search input[type=search]{width:100%}.wy-dropdown-menu>dd.call-to-action{background:#e3e3e3;text-transform:uppercase;font-weight:500;font-size:80%}.wy-dropdown-menu>dd.call-to-action:hover{background:#e3e3e3}.wy-dropdown-menu>dd.call-to-action .btn{color:#fff}.wy-dropdown.wy-dropdown-up .wy-dropdown-menu{bottom:100%;top:auto;left:auto;right:0}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu{background:#fcfcfc;margin-top:2px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a{padding:6px 12px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover{background:#2980b9;color:#fff}.wy-dropdown.wy-dropdown-left .wy-dropdown-menu{right:0;left:auto;text-align:right}.wy-dropdown-arrow:before{content:" ";border-bottom:5px solid #f5f5f5;border-left:5px solid transparent;border-right:5px solid transparent;position:absolute;display:block;top:-4px;left:50%;margin-left:-3px}.wy-dropdown-arrow.wy-dropdown-arrow-left:before{left:11px}.wy-form-stacked select{display:block}.wy-form-aligned .wy-help-inline,.wy-form-aligned input,.wy-form-aligned label,.wy-form-aligned select,.wy-form-aligned textarea{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-form-aligned .wy-control-group>label{display:inline-block;vertical-align:middle;width:10em;margin:6px 12px 0 0;float:left}.wy-form-aligned .wy-control{float:left}.wy-form-aligned .wy-control label{display:block}.wy-form-aligned .wy-control select{margin-top:6px}fieldset{margin:0}fieldset,legend{border:0;padding:0}legend{width:100%;white-space:normal;margin-bottom:24px;font-size:150%;*margin-left:-7px}label,legend{display:block}label{margin:0 0 .3125em;color:#333;font-size:90%}input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}.wy-control-group{margin-bottom:24px;max-width:1200px;margin-left:auto;margin-right:auto;*zoom:1}.wy-control-group:after,.wy-control-group:before{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group.wy-control-group-required>label:after{content:" *";color:#e74c3c}.wy-control-group .wy-form-full,.wy-control-group .wy-form-halves,.wy-control-group .wy-form-thirds{padding-bottom:12px}.wy-control-group .wy-form-full input[type=color],.wy-control-group .wy-form-full input[type=date],.wy-control-group .wy-form-full input[type=datetime-local],.wy-control-group .wy-form-full input[type=datetime],.wy-control-group .wy-form-full input[type=email],.wy-control-group .wy-form-full input[type=month],.wy-control-group .wy-form-full input[type=number],.wy-control-group .wy-form-full input[type=password],.wy-control-group .wy-form-full input[type=search],.wy-control-group .wy-form-full input[type=tel],.wy-control-group .wy-form-full input[type=text],.wy-control-group .wy-form-full input[type=time],.wy-control-group .wy-form-full input[type=url],.wy-control-group .wy-form-full input[type=week],.wy-control-group .wy-form-full select,.wy-control-group .wy-form-halves input[type=color],.wy-control-group .wy-form-halves input[type=date],.wy-control-group .wy-form-halves input[type=datetime-local],.wy-control-group .wy-form-halves input[type=datetime],.wy-control-group .wy-form-halves input[type=email],.wy-control-group .wy-form-halves input[type=month],.wy-control-group .wy-form-halves input[type=number],.wy-control-group .wy-form-halves input[type=password],.wy-control-group .wy-form-halves input[type=search],.wy-control-group .wy-form-halves input[type=tel],.wy-control-group .wy-form-halves input[type=text],.wy-control-group .wy-form-halves input[type=time],.wy-control-group .wy-form-halves input[type=url],.wy-control-group .wy-form-halves input[type=week],.wy-control-group .wy-form-halves select,.wy-control-group .wy-form-thirds input[type=color],.wy-control-group .wy-form-thirds input[type=date],.wy-control-group .wy-form-thirds input[type=datetime-local],.wy-control-group .wy-form-thirds input[type=datetime],.wy-control-group .wy-form-thirds input[type=email],.wy-control-group .wy-form-thirds input[type=month],.wy-control-group .wy-form-thirds input[type=number],.wy-control-group .wy-form-thirds input[type=password],.wy-control-group .wy-form-thirds input[type=search],.wy-control-group .wy-form-thirds input[type=tel],.wy-control-group .wy-form-thirds input[type=text],.wy-control-group .wy-form-thirds input[type=time],.wy-control-group .wy-form-thirds input[type=url],.wy-control-group .wy-form-thirds input[type=week],.wy-control-group .wy-form-thirds select{width:100%}.wy-control-group .wy-form-full{float:left;display:block;width:100%;margin-right:0}.wy-control-group .wy-form-full:last-child{margin-right:0}.wy-control-group .wy-form-halves{float:left;display:block;margin-right:2.35765%;width:48.82117%}.wy-control-group .wy-form-halves:last-child,.wy-control-group .wy-form-halves:nth-of-type(2n){margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(odd){clear:left}.wy-control-group .wy-form-thirds{float:left;display:block;margin-right:2.35765%;width:31.76157%}.wy-control-group .wy-form-thirds:last-child,.wy-control-group .wy-form-thirds:nth-of-type(3n){margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n+1){clear:left}.wy-control-group.wy-control-group-no-input .wy-control,.wy-control-no-input{margin:6px 0 0;font-size:90%}.wy-control-no-input{display:inline-block}.wy-control-group.fluid-input input[type=color],.wy-control-group.fluid-input input[type=date],.wy-control-group.fluid-input input[type=datetime-local],.wy-control-group.fluid-input input[type=datetime],.wy-control-group.fluid-input input[type=email],.wy-control-group.fluid-input input[type=month],.wy-control-group.fluid-input input[type=number],.wy-control-group.fluid-input input[type=password],.wy-control-group.fluid-input input[type=search],.wy-control-group.fluid-input input[type=tel],.wy-control-group.fluid-input input[type=text],.wy-control-group.fluid-input input[type=time],.wy-control-group.fluid-input input[type=url],.wy-control-group.fluid-input input[type=week]{width:100%}.wy-form-message-inline{padding-left:.3em;color:#666;font-size:90%}.wy-form-message{display:block;color:#999;font-size:70%;margin-top:.3125em;font-style:italic}.wy-form-message p{font-size:inherit;font-style:italic;margin-bottom:6px}.wy-form-message p:last-child{margin-bottom:0}input{line-height:normal}input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;*overflow:visible}input[type=color],input[type=date],input[type=datetime-local],input[type=datetime],input[type=email],input[type=month],input[type=number],input[type=password],input[type=search],input[type=tel],input[type=text],input[type=time],input[type=url],input[type=week]{-webkit-appearance:none;padding:6px;display:inline-block;border:1px solid #ccc;font-size:80%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;box-shadow:inset 0 1px 3px #ddd;border-radius:0;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}input[type=datetime-local]{padding:.34375em .625em}input[disabled]{cursor:default}input[type=checkbox],input[type=radio]{padding:0;margin-right:.3125em;*height:13px;*width:13px}input[type=checkbox],input[type=radio],input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}input[type=color]:focus,input[type=date]:focus,input[type=datetime-local]:focus,input[type=datetime]:focus,input[type=email]:focus,input[type=month]:focus,input[type=number]:focus,input[type=password]:focus,input[type=search]:focus,input[type=tel]:focus,input[type=text]:focus,input[type=time]:focus,input[type=url]:focus,input[type=week]:focus{outline:0;outline:thin dotted\9;border-color:#333}input.no-focus:focus{border-color:#ccc!important}input[type=checkbox]:focus,input[type=file]:focus,input[type=radio]:focus{outline:thin dotted #333;outline:1px auto #129fea}input[type=color][disabled],input[type=date][disabled],input[type=datetime-local][disabled],input[type=datetime][disabled],input[type=email][disabled],input[type=month][disabled],input[type=number][disabled],input[type=password][disabled],input[type=search][disabled],input[type=tel][disabled],input[type=text][disabled],input[type=time][disabled],input[type=url][disabled],input[type=week][disabled]{cursor:not-allowed;background-color:#fafafa}input:focus:invalid,select:focus:invalid,textarea:focus:invalid{color:#e74c3c;border:1px solid #e74c3c}input:focus:invalid:focus,select:focus:invalid:focus,textarea:focus:invalid:focus{border-color:#e74c3c}input[type=checkbox]:focus:invalid:focus,input[type=file]:focus:invalid:focus,input[type=radio]:focus:invalid:focus{outline-color:#e74c3c}input.wy-input-large{padding:12px;font-size:100%}textarea{overflow:auto;vertical-align:top;width:100%;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif}select,textarea{padding:.5em .625em;display:inline-block;border:1px solid #ccc;font-size:80%;box-shadow:inset 0 1px 3px #ddd;-webkit-transition:border .3s linear;-moz-transition:border .3s linear;transition:border .3s linear}select{border:1px solid #ccc;background-color:#fff}select[multiple]{height:auto}select:focus,textarea:focus{outline:0}input[readonly],select[disabled],select[readonly],textarea[disabled],textarea[readonly]{cursor:not-allowed;background-color:#fafafa}input[type=checkbox][disabled],input[type=radio][disabled]{cursor:not-allowed}.wy-checkbox,.wy-radio{margin:6px 0;color:#404040;display:block}.wy-checkbox input,.wy-radio input{vertical-align:baseline}.wy-form-message-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-input-prefix,.wy-input-suffix{white-space:nowrap;padding:6px}.wy-input-prefix .wy-input-context,.wy-input-suffix .wy-input-context{line-height:27px;padding:0 8px;display:inline-block;font-size:80%;background-color:#f3f6f6;border:1px solid #ccc;color:#999}.wy-input-suffix .wy-input-context{border-left:0}.wy-input-prefix .wy-input-context{border-right:0}.wy-switch{position:relative;display:block;height:24px;margin-top:12px;cursor:pointer}.wy-switch:before{left:0;top:0;width:36px;height:12px;background:#ccc}.wy-switch:after,.wy-switch:before{position:absolute;content:"";display:block;border-radius:4px;-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.wy-switch:after{width:18px;height:18px;background:#999;left:-3px;top:-3px}.wy-switch span{position:absolute;left:48px;display:block;font-size:12px;color:#ccc;line-height:1}.wy-switch.active:before{background:#1e8449}.wy-switch.active:after{left:24px;background:#27ae60}.wy-switch.disabled{cursor:not-allowed;opacity:.8}.wy-control-group.wy-control-group-error .wy-form-message,.wy-control-group.wy-control-group-error>label{color:#e74c3c}.wy-control-group.wy-control-group-error input[type=color],.wy-control-group.wy-control-group-error input[type=date],.wy-control-group.wy-control-group-error input[type=datetime-local],.wy-control-group.wy-control-group-error input[type=datetime],.wy-control-group.wy-control-group-error input[type=email],.wy-control-group.wy-control-group-error input[type=month],.wy-control-group.wy-control-group-error input[type=number],.wy-control-group.wy-control-group-error input[type=password],.wy-control-group.wy-control-group-error input[type=search],.wy-control-group.wy-control-group-error input[type=tel],.wy-control-group.wy-control-group-error input[type=text],.wy-control-group.wy-control-group-error input[type=time],.wy-control-group.wy-control-group-error input[type=url],.wy-control-group.wy-control-group-error input[type=week],.wy-control-group.wy-control-group-error textarea{border:1px solid #e74c3c}.wy-inline-validate{white-space:nowrap}.wy-inline-validate .wy-input-context{padding:.5em .625em;display:inline-block;font-size:80%}.wy-inline-validate.wy-inline-validate-success .wy-input-context{color:#27ae60}.wy-inline-validate.wy-inline-validate-danger .wy-input-context{color:#e74c3c}.wy-inline-validate.wy-inline-validate-warning .wy-input-context{color:#e67e22}.wy-inline-validate.wy-inline-validate-info .wy-input-context{color:#2980b9}.rotate-90{-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.rotate-180{-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.rotate-270{-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.mirror{-webkit-transform:scaleX(-1);-moz-transform:scaleX(-1);-ms-transform:scaleX(-1);-o-transform:scaleX(-1);transform:scaleX(-1)}.mirror.rotate-90{-webkit-transform:scaleX(-1) rotate(90deg);-moz-transform:scaleX(-1) rotate(90deg);-ms-transform:scaleX(-1) rotate(90deg);-o-transform:scaleX(-1) rotate(90deg);transform:scaleX(-1) rotate(90deg)}.mirror.rotate-180{-webkit-transform:scaleX(-1) rotate(180deg);-moz-transform:scaleX(-1) rotate(180deg);-ms-transform:scaleX(-1) rotate(180deg);-o-transform:scaleX(-1) rotate(180deg);transform:scaleX(-1) rotate(180deg)}.mirror.rotate-270{-webkit-transform:scaleX(-1) rotate(270deg);-moz-transform:scaleX(-1) rotate(270deg);-ms-transform:scaleX(-1) rotate(270deg);-o-transform:scaleX(-1) rotate(270deg);transform:scaleX(-1) rotate(270deg)}@media only screen and (max-width:480px){.wy-form button[type=submit]{margin:.7em 0 0}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=text],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week],.wy-form label{margin-bottom:.3em;display:block}.wy-form input[type=color],.wy-form input[type=date],.wy-form input[type=datetime-local],.wy-form input[type=datetime],.wy-form input[type=email],.wy-form input[type=month],.wy-form input[type=number],.wy-form input[type=password],.wy-form input[type=search],.wy-form input[type=tel],.wy-form input[type=time],.wy-form input[type=url],.wy-form input[type=week]{margin-bottom:0}.wy-form-aligned .wy-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.wy-form-aligned .wy-control{margin:1.5em 0 0}.wy-form-message,.wy-form-message-inline,.wy-form .wy-help-inline{display:block;font-size:80%;padding:6px 0}}@media screen and (max-width:768px){.tablet-hide{display:none}}@media screen and (max-width:480px){.mobile-hide{display:none}}.float-left{float:left}.float-right{float:right}.full-width{width:100%}.rst-content table.docutils,.rst-content table.field-list,.wy-table{border-collapse:collapse;border-spacing:0;empty-cells:show;margin-bottom:24px}.rst-content table.docutils caption,.rst-content table.field-list caption,.wy-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.rst-content table.docutils td,.rst-content table.docutils th,.rst-content table.field-list td,.rst-content table.field-list th,.wy-table td,.wy-table th{font-size:90%;margin:0;overflow:visible;padding:8px 16px}.rst-content table.docutils td:first-child,.rst-content table.docutils th:first-child,.rst-content table.field-list td:first-child,.rst-content table.field-list th:first-child,.wy-table td:first-child,.wy-table th:first-child{border-left-width:0}.rst-content table.docutils thead,.rst-content table.field-list thead,.wy-table thead{color:#000;text-align:left;vertical-align:bottom;white-space:nowrap}.rst-content table.docutils thead th,.rst-content table.field-list thead th,.wy-table thead th{font-weight:700;border-bottom:2px solid #e1e4e5}.rst-content table.docutils td,.rst-content table.field-list td,.wy-table td{background-color:transparent;vertical-align:middle}.rst-content table.docutils td p,.rst-content table.field-list td p,.wy-table td p{line-height:18px}.rst-content table.docutils td p:last-child,.rst-content table.field-list td p:last-child,.wy-table td p:last-child{margin-bottom:0}.rst-content table.docutils .wy-table-cell-min,.rst-content table.field-list .wy-table-cell-min,.wy-table .wy-table-cell-min{width:1%;padding-right:0}.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox],.wy-table .wy-table-cell-min input[type=checkbox]{margin:0}.wy-table-secondary{color:grey;font-size:90%}.wy-table-tertiary{color:grey;font-size:80%}.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,.wy-table-backed,.wy-table-odd td,.wy-table-striped tr:nth-child(2n-1) td{background-color:#f3f6f6}.rst-content table.docutils,.wy-table-bordered-all{border:1px solid #e1e4e5}.rst-content table.docutils td,.wy-table-bordered-all td{border-bottom:1px solid #e1e4e5;border-left:1px solid #e1e4e5}.rst-content table.docutils tbody>tr:last-child td,.wy-table-bordered-all tbody>tr:last-child td{border-bottom-width:0}.wy-table-bordered{border:1px solid #e1e4e5}.wy-table-bordered-rows td{border-bottom:1px solid #e1e4e5}.wy-table-bordered-rows tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal td,.wy-table-horizontal th{border-width:0 0 1px;border-bottom:1px solid #e1e4e5}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-responsive{margin-bottom:24px;max-width:100%;overflow:auto}.wy-table-responsive table{margin-bottom:0!important}.wy-table-responsive table td,.wy-table-responsive table th{white-space:nowrap}a{color:#2980b9;text-decoration:none;cursor:pointer}a:hover{color:#3091d1}a:visited{color:#9b59b6}html{height:100%}body,html{overflow-x:hidden}body{font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-weight:400;color:#404040;min-height:100%;background:#edf0f2}.wy-text-left{text-align:left}.wy-text-center{text-align:center}.wy-text-right{text-align:right}.wy-text-large{font-size:120%}.wy-text-normal{font-size:100%}.wy-text-small,small{font-size:80%}.wy-text-strike{text-decoration:line-through}.wy-text-warning{color:#e67e22!important}a.wy-text-warning:hover{color:#eb9950!important}.wy-text-info{color:#2980b9!important}a.wy-text-info:hover{color:#409ad5!important}.wy-text-success{color:#27ae60!important}a.wy-text-success:hover{color:#36d278!important}.wy-text-danger{color:#e74c3c!important}a.wy-text-danger:hover{color:#ed7669!important}.wy-text-neutral{color:#404040!important}a.wy-text-neutral:hover{color:#595959!important}.rst-content .toctree-wrapper>p.caption,h1,h2,h3,h4,h5,h6,legend{margin-top:0;font-weight:700;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif}p{line-height:24px;font-size:16px;margin:0 0 24px}h1{font-size:175%}.rst-content .toctree-wrapper>p.caption,h2{font-size:150%}h3{font-size:125%}h4{font-size:115%}h5{font-size:110%}h6{font-size:100%}hr{display:block;height:1px;border:0;border-top:1px solid #e1e4e5;margin:24px 0;padding:0}.rst-content code,.rst-content tt,code{white-space:nowrap;max-width:100%;background:#fff;border:1px solid #e1e4e5;font-size:75%;padding:0 5px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#e74c3c;overflow-x:auto}.rst-content tt.code-large,code.code-large{font-size:90%}.rst-content .section ul,.rst-content .toctree-wrapper ul,.rst-content section ul,.wy-plain-list-disc,article ul{list-style:disc;line-height:24px;margin-bottom:24px}.rst-content .section ul li,.rst-content .toctree-wrapper ul li,.rst-content section ul li,.wy-plain-list-disc li,article ul li{list-style:disc;margin-left:24px}.rst-content .section ul li p:last-child,.rst-content .section ul li ul,.rst-content .toctree-wrapper ul li p:last-child,.rst-content .toctree-wrapper ul li ul,.rst-content section ul li p:last-child,.rst-content section ul li ul,.wy-plain-list-disc li p:last-child,.wy-plain-list-disc li ul,article ul li p:last-child,article ul li ul{margin-bottom:0}.rst-content .section ul li li,.rst-content .toctree-wrapper ul li li,.rst-content section ul li li,.wy-plain-list-disc li li,article ul li li{list-style:circle}.rst-content .section ul li li li,.rst-content .toctree-wrapper ul li li li,.rst-content section ul li li li,.wy-plain-list-disc li li li,article ul li li li{list-style:square}.rst-content .section ul li ol li,.rst-content .toctree-wrapper ul li ol li,.rst-content section ul li ol li,.wy-plain-list-disc li ol li,article ul li ol li{list-style:decimal}.rst-content .section ol,.rst-content .section ol.arabic,.rst-content .toctree-wrapper ol,.rst-content .toctree-wrapper ol.arabic,.rst-content section ol,.rst-content section ol.arabic,.wy-plain-list-decimal,article ol{list-style:decimal;line-height:24px;margin-bottom:24px}.rst-content .section ol.arabic li,.rst-content .section ol li,.rst-content .toctree-wrapper ol.arabic li,.rst-content .toctree-wrapper ol li,.rst-content section ol.arabic li,.rst-content section ol li,.wy-plain-list-decimal li,article ol li{list-style:decimal;margin-left:24px}.rst-content .section ol.arabic li ul,.rst-content .section ol li p:last-child,.rst-content .section ol li ul,.rst-content .toctree-wrapper ol.arabic li ul,.rst-content .toctree-wrapper ol li p:last-child,.rst-content .toctree-wrapper ol li ul,.rst-content section ol.arabic li ul,.rst-content section ol li p:last-child,.rst-content section ol li ul,.wy-plain-list-decimal li p:last-child,.wy-plain-list-decimal li ul,article ol li p:last-child,article ol li ul{margin-bottom:0}.rst-content .section ol.arabic li ul li,.rst-content .section ol li ul li,.rst-content .toctree-wrapper ol.arabic li ul li,.rst-content .toctree-wrapper ol li ul li,.rst-content section ol.arabic li ul li,.rst-content section ol li ul li,.wy-plain-list-decimal li ul li,article ol li ul li{list-style:disc}.wy-breadcrumbs{*zoom:1}.wy-breadcrumbs:after,.wy-breadcrumbs:before{display:table;content:""}.wy-breadcrumbs:after{clear:both}.wy-breadcrumbs>li{display:inline-block;padding-top:5px}.wy-breadcrumbs>li.wy-breadcrumbs-aside{float:right}.rst-content .wy-breadcrumbs>li code,.rst-content .wy-breadcrumbs>li tt,.wy-breadcrumbs>li .rst-content tt,.wy-breadcrumbs>li code{all:inherit;color:inherit}.breadcrumb-item:before{content:"/";color:#bbb;font-size:13px;padding:0 6px 0 3px}.wy-breadcrumbs-extra{margin-bottom:0;color:#b3b3b3;font-size:80%;display:inline-block}@media screen and (max-width:480px){.wy-breadcrumbs-extra,.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}@media print{.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}html{font-size:16px}.wy-affix{position:fixed;top:1.618em}.wy-menu a:hover{text-decoration:none}.wy-menu-horiz{*zoom:1}.wy-menu-horiz:after,.wy-menu-horiz:before{display:table;content:""}.wy-menu-horiz:after{clear:both}.wy-menu-horiz li,.wy-menu-horiz ul{display:inline-block}.wy-menu-horiz li:hover{background:hsla(0,0%,100%,.1)}.wy-menu-horiz li.divide-left{border-left:1px solid #404040}.wy-menu-horiz li.divide-right{border-right:1px solid #404040}.wy-menu-horiz a{height:32px;display:inline-block;line-height:32px;padding:0 16px}.wy-menu-vertical{width:300px}.wy-menu-vertical header,.wy-menu-vertical p.caption{color:#55a5d9;height:32px;line-height:32px;padding:0 1.618em;margin:12px 0 0;display:block;font-weight:700;text-transform:uppercase;font-size:85%;white-space:nowrap}.wy-menu-vertical ul{margin-bottom:0}.wy-menu-vertical li.divide-top{border-top:1px solid #404040}.wy-menu-vertical li.divide-bottom{border-bottom:1px solid #404040}.wy-menu-vertical li.current{background:#e3e3e3}.wy-menu-vertical li.current a{color:grey;border-right:1px solid #c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.current a:hover{background:#d6d6d6}.rst-content .wy-menu-vertical li tt,.wy-menu-vertical li .rst-content tt,.wy-menu-vertical li code{border:none;background:inherit;color:inherit;padding-left:0;padding-right:0}.wy-menu-vertical li button.toctree-expand{display:block;float:left;margin-left:-1.2em;line-height:18px;color:#4d4d4d;border:none;background:none;padding:0}.wy-menu-vertical li.current>a,.wy-menu-vertical li.on a{color:#404040;font-weight:700;position:relative;background:#fcfcfc;border:none;padding:.4045em 1.618em}.wy-menu-vertical li.current>a:hover,.wy-menu-vertical li.on a:hover{background:#fcfcfc}.wy-menu-vertical li.current>a:hover button.toctree-expand,.wy-menu-vertical li.on a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.current>a button.toctree-expand,.wy-menu-vertical li.on a button.toctree-expand{display:block;line-height:18px;color:#333}.wy-menu-vertical li.toctree-l1.current>a{border-bottom:1px solid #c9c9c9;border-top:1px solid #c9c9c9}.wy-menu-vertical .toctree-l1.current .toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .toctree-l11>ul{display:none}.wy-menu-vertical .toctree-l1.current .current.toctree-l2>ul,.wy-menu-vertical .toctree-l2.current .current.toctree-l3>ul,.wy-menu-vertical .toctree-l3.current .current.toctree-l4>ul,.wy-menu-vertical .toctree-l4.current .current.toctree-l5>ul,.wy-menu-vertical .toctree-l5.current .current.toctree-l6>ul,.wy-menu-vertical .toctree-l6.current .current.toctree-l7>ul,.wy-menu-vertical .toctree-l7.current .current.toctree-l8>ul,.wy-menu-vertical .toctree-l8.current .current.toctree-l9>ul,.wy-menu-vertical .toctree-l9.current .current.toctree-l10>ul,.wy-menu-vertical .toctree-l10.current .current.toctree-l11>ul{display:block}.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4{font-size:.9em}.wy-menu-vertical li.toctree-l2 a,.wy-menu-vertical li.toctree-l3 a,.wy-menu-vertical li.toctree-l4 a,.wy-menu-vertical li.toctree-l5 a,.wy-menu-vertical li.toctree-l6 a,.wy-menu-vertical li.toctree-l7 a,.wy-menu-vertical li.toctree-l8 a,.wy-menu-vertical li.toctree-l9 a,.wy-menu-vertical li.toctree-l10 a{color:#404040}.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l4 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l5 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l6 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l7 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l8 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l9 a:hover button.toctree-expand,.wy-menu-vertical li.toctree-l10 a:hover button.toctree-expand{color:grey}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a,.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a,.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a,.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a,.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a,.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a,.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a,.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{display:block}.wy-menu-vertical li.toctree-l2.current>a{padding:.4045em 2.427em}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{padding:.4045em 1.618em .4045em 4.045em}.wy-menu-vertical li.toctree-l3.current>a{padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{padding:.4045em 1.618em .4045em 5.663em}.wy-menu-vertical li.toctree-l4.current>a{padding:.4045em 5.663em}.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a{padding:.4045em 1.618em .4045em 7.281em}.wy-menu-vertical li.toctree-l5.current>a{padding:.4045em 7.281em}.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a{padding:.4045em 1.618em .4045em 8.899em}.wy-menu-vertical li.toctree-l6.current>a{padding:.4045em 8.899em}.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a{padding:.4045em 1.618em .4045em 10.517em}.wy-menu-vertical li.toctree-l7.current>a{padding:.4045em 10.517em}.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a{padding:.4045em 1.618em .4045em 12.135em}.wy-menu-vertical li.toctree-l8.current>a{padding:.4045em 12.135em}.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a{padding:.4045em 1.618em .4045em 13.753em}.wy-menu-vertical li.toctree-l9.current>a{padding:.4045em 13.753em}.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a{padding:.4045em 1.618em .4045em 15.371em}.wy-menu-vertical li.toctree-l10.current>a{padding:.4045em 15.371em}.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a{padding:.4045em 1.618em .4045em 16.989em}.wy-menu-vertical li.toctree-l2.current>a,.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{background:#c9c9c9}.wy-menu-vertical li.toctree-l2 button.toctree-expand{color:#a3a3a3}.wy-menu-vertical li.toctree-l3.current>a,.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{background:#bdbdbd}.wy-menu-vertical li.toctree-l3 button.toctree-expand{color:#969696}.wy-menu-vertical li.current ul{display:block}.wy-menu-vertical li ul{margin-bottom:0;display:none}.wy-menu-vertical li ul li a{margin-bottom:0;color:#d9d9d9;font-weight:400}.wy-menu-vertical a{line-height:18px;padding:.4045em 1.618em;display:block;position:relative;font-size:90%;color:#d9d9d9}.wy-menu-vertical a:hover{background-color:#4e4a4a;cursor:pointer}.wy-menu-vertical a:hover button.toctree-expand{color:#d9d9d9}.wy-menu-vertical a:active{background-color:#2980b9;cursor:pointer;color:#fff}.wy-menu-vertical a:active button.toctree-expand{color:#fff}.wy-side-nav-search{display:block;width:300px;padding:.809em;margin-bottom:.809em;z-index:200;background-color:#2980b9;text-align:center;color:#fcfcfc}.wy-side-nav-search input[type=text]{width:100%;border-radius:50px;padding:6px 12px;border-color:#2472a4}.wy-side-nav-search img{display:block;margin:auto auto .809em;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-side-nav-search .wy-dropdown>a,.wy-side-nav-search>a{color:#fcfcfc;font-size:100%;font-weight:700;display:inline-block;padding:4px 6px;margin-bottom:.809em;max-width:100%}.wy-side-nav-search .wy-dropdown>a:hover,.wy-side-nav-search .wy-dropdown>aactive,.wy-side-nav-search .wy-dropdown>afocus,.wy-side-nav-search>a:hover,.wy-side-nav-search>aactive,.wy-side-nav-search>afocus{background:hsla(0,0%,100%,.1)}.wy-side-nav-search .wy-dropdown>a img.logo,.wy-side-nav-search>a img.logo{display:block;margin:0 auto;height:auto;width:auto;border-radius:0;max-width:100%;background:transparent}.wy-side-nav-search .wy-dropdown>a.icon,.wy-side-nav-search>a.icon{display:block}.wy-side-nav-search .wy-dropdown>a.icon img.logo,.wy-side-nav-search>a.icon img.logo{margin-top:.85em}.wy-side-nav-search>div.switch-menus{position:relative;display:block;margin-top:-.4045em;margin-bottom:.809em;font-weight:400;color:hsla(0,0%,100%,.3)}.wy-side-nav-search>div.switch-menus>div.language-switch,.wy-side-nav-search>div.switch-menus>div.version-switch{display:inline-block;padding:.2em}.wy-side-nav-search>div.switch-menus>div.language-switch select,.wy-side-nav-search>div.switch-menus>div.version-switch select{display:inline-block;margin-right:-2rem;padding-right:2rem;max-width:240px;text-align-last:center;background:none;border:none;border-radius:0;box-shadow:none;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;font-size:1em;font-weight:400;color:hsla(0,0%,100%,.3);cursor:pointer;appearance:none;-webkit-appearance:none;-moz-appearance:none}.wy-side-nav-search>div.switch-menus>div.language-switch select:active,.wy-side-nav-search>div.switch-menus>div.language-switch select:focus,.wy-side-nav-search>div.switch-menus>div.language-switch select:hover,.wy-side-nav-search>div.switch-menus>div.version-switch select:active,.wy-side-nav-search>div.switch-menus>div.version-switch select:focus,.wy-side-nav-search>div.switch-menus>div.version-switch select:hover{background:hsla(0,0%,100%,.1);color:hsla(0,0%,100%,.5)}.wy-side-nav-search>div.switch-menus>div.language-switch select option,.wy-side-nav-search>div.switch-menus>div.version-switch select option{color:#000}.wy-side-nav-search>div.switch-menus>div.language-switch:has(>select):after,.wy-side-nav-search>div.switch-menus>div.version-switch:has(>select):after{display:inline-block;width:1.5em;height:100%;padding:.1em;content:"\f0d7";font-size:1em;line-height:1.2em;font-family:FontAwesome;text-align:center;pointer-events:none;box-sizing:border-box}.wy-nav .wy-menu-vertical header{color:#2980b9}.wy-nav .wy-menu-vertical a{color:#b3b3b3}.wy-nav .wy-menu-vertical a:hover{background-color:#2980b9;color:#fff}[data-menu-wrap]{-webkit-transition:all .2s ease-in;-moz-transition:all .2s ease-in;transition:all .2s ease-in;position:absolute;opacity:1;width:100%;opacity:0}[data-menu-wrap].move-center{left:0;right:auto;opacity:1}[data-menu-wrap].move-left{right:auto;left:-100%;opacity:0}[data-menu-wrap].move-right{right:-100%;left:auto;opacity:0}.wy-body-for-nav{background:#fcfcfc}.wy-grid-for-nav{position:absolute;width:100%;height:100%}.wy-nav-side{position:fixed;top:0;bottom:0;left:0;padding-bottom:2em;width:300px;overflow-x:hidden;overflow-y:hidden;min-height:100%;color:#9b9b9b;background:#343131;z-index:200}.wy-side-scroll{width:320px;position:relative;overflow-x:hidden;overflow-y:scroll;height:100%}.wy-nav-top{display:none;background:#2980b9;color:#fff;padding:.4045em .809em;position:relative;line-height:50px;text-align:center;font-size:100%;*zoom:1}.wy-nav-top:after,.wy-nav-top:before{display:table;content:""}.wy-nav-top:after{clear:both}.wy-nav-top a{color:#fff;font-weight:700}.wy-nav-top img{margin-right:12px;height:45px;width:45px;background-color:#2980b9;padding:5px;border-radius:100%}.wy-nav-top i{font-size:30px;float:left;cursor:pointer;padding-top:inherit}.wy-nav-content-wrap{margin-left:300px;background:#fcfcfc;min-height:100%}.wy-nav-content{padding:1.618em 3.236em;height:100%;max-width:800px;margin:auto}.wy-body-mask{position:fixed;width:100%;height:100%;background:rgba(0,0,0,.2);display:none;z-index:499}.wy-body-mask.on{display:block}footer{color:grey}footer p{margin-bottom:12px}.rst-content footer span.commit tt,footer span.commit .rst-content tt,footer span.commit code{padding:0;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:1em;background:none;border:none;color:grey}.rst-footer-buttons{*zoom:1}.rst-footer-buttons:after,.rst-footer-buttons:before{width:100%;display:table;content:""}.rst-footer-buttons:after{clear:both}.rst-breadcrumbs-buttons{margin-top:12px;*zoom:1}.rst-breadcrumbs-buttons:after,.rst-breadcrumbs-buttons:before{display:table;content:""}.rst-breadcrumbs-buttons:after{clear:both}#search-results .search li{margin-bottom:24px;border-bottom:1px solid #e1e4e5;padding-bottom:24px}#search-results .search li:first-child{border-top:1px solid #e1e4e5;padding-top:24px}#search-results .search li a{font-size:120%;margin-bottom:12px;display:inline-block}#search-results .context{color:grey;font-size:90%}.genindextable li>ul{margin-left:24px}@media screen and (max-width:768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}.wy-nav-side{left:-300px}.wy-nav-side.shift{width:85%;left:0}.wy-menu.wy-menu-vertical,.wy-side-nav-search,.wy-side-scroll{width:auto}.wy-nav-content-wrap{margin-left:0}.wy-nav-content-wrap .wy-nav-content{padding:1.618em}.wy-nav-content-wrap.shift{position:fixed;min-width:100%;left:85%;top:0;height:100%;overflow:hidden}}@media screen and (min-width:1100px){.wy-nav-content-wrap{background:rgba(0,0,0,.05)}.wy-nav-content{margin:0;background:#fcfcfc}}@media print{.rst-versions,.wy-nav-side,footer{display:none}.wy-nav-content-wrap{margin-left:0}}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60;*zoom:1}.rst-versions .rst-current-version:after,.rst-versions .rst-current-version:before{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-content .code-block-caption .rst-versions .rst-current-version .headerlink,.rst-content .eqno .rst-versions .rst-current-version .headerlink,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-content code.download .rst-versions .rst-current-version span:first-child,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-content p .rst-versions .rst-current-version .headerlink,.rst-content table>caption .rst-versions .rst-current-version .headerlink,.rst-content tt.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .icon,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-versions .rst-current-version .rst-content .code-block-caption .headerlink,.rst-versions .rst-current-version .rst-content .eqno .headerlink,.rst-versions .rst-current-version .rst-content code.download span:first-child,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-versions .rst-current-version .rst-content p .headerlink,.rst-versions .rst-current-version .rst-content table>caption .headerlink,.rst-versions .rst-current-version .rst-content tt.download span:first-child,.rst-versions .rst-current-version .wy-menu-vertical li button.toctree-expand,.wy-menu-vertical li .rst-versions .rst-current-version button.toctree-expand{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}.rst-content .toctree-wrapper>p.caption,.rst-content h1,.rst-content h2,.rst-content h3,.rst-content h4,.rst-content h5,.rst-content h6{margin-bottom:24px}.rst-content img{max-width:100%;height:auto}.rst-content div.figure,.rst-content figure{margin-bottom:24px}.rst-content div.figure .caption-text,.rst-content figure .caption-text{font-style:italic}.rst-content div.figure p:last-child.caption,.rst-content figure p:last-child.caption{margin-bottom:0}.rst-content div.figure.align-center,.rst-content figure.align-center{text-align:center}.rst-content .section>a>img,.rst-content .section>img,.rst-content section>a>img,.rst-content section>img{margin-bottom:24px}.rst-content abbr[title]{text-decoration:none}.rst-content.style-external-links a.reference.external:after{font-family:FontAwesome;content:"\f08e";color:#b3b3b3;vertical-align:super;font-size:60%;margin:0 .2em}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content pre.literal-block{white-space:pre;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;display:block;overflow:auto}.rst-content div[class^=highlight],.rst-content pre.literal-block{border:1px solid #e1e4e5;overflow-x:auto;margin:1px 0 24px}.rst-content div[class^=highlight] div[class^=highlight],.rst-content pre.literal-block div[class^=highlight]{padding:0;border:none;margin:0}.rst-content div[class^=highlight] td.code{width:100%}.rst-content .linenodiv pre{border-right:1px solid #e6e9ea;margin:0;padding:12px;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;user-select:none;pointer-events:none}.rst-content div[class^=highlight] pre{white-space:pre;margin:0;padding:12px;display:block;overflow:auto}.rst-content div[class^=highlight] pre .hll{display:block;margin:0 -12px;padding:0 12px}.rst-content .linenodiv pre,.rst-content div[class^=highlight] pre,.rst-content pre.literal-block{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;font-size:12px;line-height:1.4}.rst-content div.highlight .gp,.rst-content div.highlight span.linenos{user-select:none;pointer-events:none}.rst-content div.highlight span.linenos{display:inline-block;padding-left:0;padding-right:12px;margin-right:12px;border-right:1px solid #e6e9ea}.rst-content .code-block-caption{font-style:italic;font-size:85%;line-height:1;padding:1em 0;text-align:center}@media print{.rst-content .codeblock,.rst-content div[class^=highlight],.rst-content div[class^=highlight] pre{white-space:pre-wrap}}.rst-content .admonition,.rst-content .admonition-todo,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .note,.rst-content .seealso,.rst-content .tip,.rst-content .warning{clear:both}.rst-content .admonition-todo .last,.rst-content .admonition-todo>:last-child,.rst-content .admonition .last,.rst-content .admonition>:last-child,.rst-content .attention .last,.rst-content .attention>:last-child,.rst-content .caution .last,.rst-content .caution>:last-child,.rst-content .danger .last,.rst-content .danger>:last-child,.rst-content .error .last,.rst-content .error>:last-child,.rst-content .hint .last,.rst-content .hint>:last-child,.rst-content .important .last,.rst-content .important>:last-child,.rst-content .note .last,.rst-content .note>:last-child,.rst-content .seealso .last,.rst-content .seealso>:last-child,.rst-content .tip .last,.rst-content .tip>:last-child,.rst-content .warning .last,.rst-content .warning>:last-child{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent!important;border-color:rgba(0,0,0,.1)!important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha>li,.rst-content .toctree-wrapper ol.loweralpha,.rst-content .toctree-wrapper ol.loweralpha>li,.rst-content section ol.loweralpha,.rst-content section ol.loweralpha>li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha>li,.rst-content .toctree-wrapper ol.upperalpha,.rst-content .toctree-wrapper ol.upperalpha>li,.rst-content section ol.upperalpha,.rst-content section ol.upperalpha>li{list-style:upper-alpha}.rst-content .section ol li>*,.rst-content .section ul li>*,.rst-content .toctree-wrapper ol li>*,.rst-content .toctree-wrapper ul li>*,.rst-content section ol li>*,.rst-content section ul li>*{margin-top:12px;margin-bottom:12px}.rst-content .section ol li>:first-child,.rst-content .section ul li>:first-child,.rst-content .toctree-wrapper ol li>:first-child,.rst-content .toctree-wrapper ul li>:first-child,.rst-content section ol li>:first-child,.rst-content section ul li>:first-child{margin-top:0}.rst-content .section ol li>p,.rst-content .section ol li>p:last-child,.rst-content .section ul li>p,.rst-content .section ul li>p:last-child,.rst-content .toctree-wrapper ol li>p,.rst-content .toctree-wrapper ol li>p:last-child,.rst-content .toctree-wrapper ul li>p,.rst-content .toctree-wrapper ul li>p:last-child,.rst-content section ol li>p,.rst-content section ol li>p:last-child,.rst-content section ul li>p,.rst-content section ul li>p:last-child{margin-bottom:12px}.rst-content .section ol li>p:only-child,.rst-content .section ol li>p:only-child:last-child,.rst-content .section ul li>p:only-child,.rst-content .section ul li>p:only-child:last-child,.rst-content .toctree-wrapper ol li>p:only-child,.rst-content .toctree-wrapper ol li>p:only-child:last-child,.rst-content .toctree-wrapper ul li>p:only-child,.rst-content .toctree-wrapper ul li>p:only-child:last-child,.rst-content section ol li>p:only-child,.rst-content section ol li>p:only-child:last-child,.rst-content section ul li>p:only-child,.rst-content section ul li>p:only-child:last-child{margin-bottom:0}.rst-content .section ol li>ol,.rst-content .section ol li>ul,.rst-content .section ul li>ol,.rst-content .section ul li>ul,.rst-content .toctree-wrapper ol li>ol,.rst-content .toctree-wrapper ol li>ul,.rst-content .toctree-wrapper ul li>ol,.rst-content .toctree-wrapper ul li>ul,.rst-content section ol li>ol,.rst-content section ol li>ul,.rst-content section ul li>ol,.rst-content section ul li>ul{margin-bottom:12px}.rst-content .section ol.simple li>*,.rst-content .section ol.simple li ol,.rst-content .section ol.simple li ul,.rst-content .section ul.simple li>*,.rst-content .section ul.simple li ol,.rst-content .section ul.simple li ul,.rst-content .toctree-wrapper ol.simple li>*,.rst-content .toctree-wrapper ol.simple li ol,.rst-content .toctree-wrapper ol.simple li ul,.rst-content .toctree-wrapper ul.simple li>*,.rst-content .toctree-wrapper ul.simple li ol,.rst-content .toctree-wrapper ul.simple li ul,.rst-content section ol.simple li>*,.rst-content section ol.simple li ol,.rst-content section ol.simple li ul,.rst-content section ul.simple li>*,.rst-content section ul.simple li ol,.rst-content section ul.simple li ul{margin-top:0;margin-bottom:0}.rst-content .line-block{margin-left:0;margin-bottom:24px;line-height:24px}.rst-content .line-block .line-block{margin-left:24px;margin-bottom:0}.rst-content .topic-title{font-weight:700;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0 0 24px 24px}.rst-content .align-left{float:left;margin:0 24px 24px 0}.rst-content .align-center{margin:auto}.rst-content .align-center:not(table){display:block}.rst-content .code-block-caption .headerlink,.rst-content .eqno .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink,.rst-content dl dt .headerlink,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content p.caption .headerlink,.rst-content p .headerlink,.rst-content table>caption .headerlink{opacity:0;font-size:14px;font-family:FontAwesome;margin-left:.5em}.rst-content .code-block-caption .headerlink:focus,.rst-content .code-block-caption:hover .headerlink,.rst-content .eqno .headerlink:focus,.rst-content .eqno:hover .headerlink,.rst-content .toctree-wrapper>p.caption .headerlink:focus,.rst-content .toctree-wrapper>p.caption:hover .headerlink,.rst-content dl dt .headerlink:focus,.rst-content dl dt:hover .headerlink,.rst-content h1 .headerlink:focus,.rst-content h1:hover .headerlink,.rst-content h2 .headerlink:focus,.rst-content h2:hover .headerlink,.rst-content h3 .headerlink:focus,.rst-content h3:hover .headerlink,.rst-content h4 .headerlink:focus,.rst-content h4:hover .headerlink,.rst-content h5 .headerlink:focus,.rst-content h5:hover .headerlink,.rst-content h6 .headerlink:focus,.rst-content h6:hover .headerlink,.rst-content p.caption .headerlink:focus,.rst-content p.caption:hover .headerlink,.rst-content p .headerlink:focus,.rst-content p:hover .headerlink,.rst-content table>caption .headerlink:focus,.rst-content table>caption:hover .headerlink{opacity:1}.rst-content p a{overflow-wrap:anywhere}.rst-content .wy-table td p,.rst-content .wy-table td ul,.rst-content .wy-table th p,.rst-content .wy-table th ul,.rst-content table.docutils td p,.rst-content table.docutils td ul,.rst-content table.docutils th p,.rst-content table.docutils th ul,.rst-content table.field-list td p,.rst-content table.field-list td ul,.rst-content table.field-list th p,.rst-content table.field-list th ul{font-size:inherit}.rst-content .btn:focus{outline:2px solid}.rst-content table>caption .headerlink:after{font-size:12px}.rst-content .centered{text-align:center}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:1px solid #e1e4e5}.rst-content .sidebar dl,.rst-content .sidebar p,.rst-content .sidebar ul{font-size:90%}.rst-content .sidebar .last,.rst-content .sidebar>:last-child{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:Roboto Slab,ff-tisa-web-pro,Georgia,Arial,sans-serif;font-weight:700;background:#e1e4e5;padding:6px 12px;margin:-24px -24px 24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;box-shadow:0 0 0 2px #f1c40f;display:inline;font-weight:700}.rst-content .citation-reference,.rst-content .footnote-reference{vertical-align:baseline;position:relative;top:-.4em;line-height:0;font-size:90%}.rst-content .citation-reference>span.fn-bracket,.rst-content .footnote-reference>span.fn-bracket{display:none}.rst-content .hlist{width:100%}.rst-content dl dt span.classifier:before{content:" : "}.rst-content dl dt span.classifier-delimiter{display:none!important}html.writer-html4 .rst-content table.docutils.citation,html.writer-html4 .rst-content table.docutils.footnote{background:none;border:none}html.writer-html4 .rst-content table.docutils.citation td,html.writer-html4 .rst-content table.docutils.citation tr,html.writer-html4 .rst-content table.docutils.footnote td,html.writer-html4 .rst-content table.docutils.footnote tr{border:none;background-color:transparent!important;white-space:normal}html.writer-html4 .rst-content table.docutils.citation td.label,html.writer-html4 .rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{display:grid;grid-template-columns:auto minmax(80%,95%)}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{display:inline-grid;grid-template-columns:max-content auto}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{display:grid;grid-template-columns:auto auto minmax(.65rem,auto) minmax(40%,95%)}html.writer-html5 .rst-content aside.citation>span.label,html.writer-html5 .rst-content aside.footnote>span.label,html.writer-html5 .rst-content div.citation>span.label{grid-column-start:1;grid-column-end:2}html.writer-html5 .rst-content aside.citation>span.backrefs,html.writer-html5 .rst-content aside.footnote>span.backrefs,html.writer-html5 .rst-content div.citation>span.backrefs{grid-column-start:2;grid-column-end:3;grid-row-start:1;grid-row-end:3}html.writer-html5 .rst-content aside.citation>p,html.writer-html5 .rst-content aside.footnote>p,html.writer-html5 .rst-content div.citation>p{grid-column-start:4;grid-column-end:5}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.field-list,html.writer-html5 .rst-content dl.footnote{margin-bottom:24px}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dt{padding-left:1rem}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.field-list>dd,html.writer-html5 .rst-content dl.field-list>dt,html.writer-html5 .rst-content dl.footnote>dd,html.writer-html5 .rst-content dl.footnote>dt{margin-bottom:0}html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{font-size:.9rem}html.writer-html5 .rst-content dl.citation>dt,html.writer-html5 .rst-content dl.footnote>dt{margin:0 .5rem .5rem 0;line-height:1.2rem;word-break:break-all;font-weight:400}html.writer-html5 .rst-content dl.citation>dt>span.brackets:before,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:before{content:"["}html.writer-html5 .rst-content dl.citation>dt>span.brackets:after,html.writer-html5 .rst-content dl.footnote>dt>span.brackets:after{content:"]"}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a{word-break:keep-all}html.writer-html5 .rst-content dl.citation>dt>span.fn-backref>a:not(:first-child):before,html.writer-html5 .rst-content dl.footnote>dt>span.fn-backref>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content dl.citation>dd,html.writer-html5 .rst-content dl.footnote>dd{margin:0 0 .5rem;line-height:1.2rem}html.writer-html5 .rst-content dl.citation>dd p,html.writer-html5 .rst-content dl.footnote>dd p{font-size:.9rem}html.writer-html5 .rst-content aside.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content div.citation{padding-left:1rem;padding-right:1rem;font-size:.9rem;line-height:1.2rem}html.writer-html5 .rst-content aside.citation p,html.writer-html5 .rst-content aside.footnote p,html.writer-html5 .rst-content div.citation p{font-size:.9rem;line-height:1.2rem;margin-bottom:12px}html.writer-html5 .rst-content aside.citation span.backrefs,html.writer-html5 .rst-content aside.footnote span.backrefs,html.writer-html5 .rst-content div.citation span.backrefs{text-align:left;font-style:italic;margin-left:.65rem;word-break:break-word;word-spacing:-.1rem;max-width:5rem}html.writer-html5 .rst-content aside.citation span.backrefs>a,html.writer-html5 .rst-content aside.footnote span.backrefs>a,html.writer-html5 .rst-content div.citation span.backrefs>a{word-break:keep-all}html.writer-html5 .rst-content aside.citation span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content aside.footnote span.backrefs>a:not(:first-child):before,html.writer-html5 .rst-content div.citation span.backrefs>a:not(:first-child):before{content:" "}html.writer-html5 .rst-content aside.citation span.label,html.writer-html5 .rst-content aside.footnote span.label,html.writer-html5 .rst-content div.citation span.label{line-height:1.2rem}html.writer-html5 .rst-content aside.citation-list,html.writer-html5 .rst-content aside.footnote-list,html.writer-html5 .rst-content div.citation-list{margin-bottom:24px}html.writer-html5 .rst-content dl.option-list kbd{font-size:.9rem}.rst-content table.docutils.footnote,html.writer-html4 .rst-content table.docutils.citation,html.writer-html5 .rst-content aside.footnote,html.writer-html5 .rst-content aside.footnote-list aside.footnote,html.writer-html5 .rst-content div.citation-list>div.citation,html.writer-html5 .rst-content dl.citation,html.writer-html5 .rst-content dl.footnote{color:grey}.rst-content table.docutils.footnote code,.rst-content table.docutils.footnote tt,html.writer-html4 .rst-content table.docutils.citation code,html.writer-html4 .rst-content table.docutils.citation tt,html.writer-html5 .rst-content aside.footnote-list aside.footnote code,html.writer-html5 .rst-content aside.footnote-list aside.footnote tt,html.writer-html5 .rst-content aside.footnote code,html.writer-html5 .rst-content aside.footnote tt,html.writer-html5 .rst-content div.citation-list>div.citation code,html.writer-html5 .rst-content div.citation-list>div.citation tt,html.writer-html5 .rst-content dl.citation code,html.writer-html5 .rst-content dl.citation tt,html.writer-html5 .rst-content dl.footnote code,html.writer-html5 .rst-content dl.footnote tt{color:#555}.rst-content .wy-table-responsive.citation,.rst-content .wy-table-responsive.footnote{margin-bottom:0}.rst-content .wy-table-responsive.citation+:not(.citation),.rst-content .wy-table-responsive.footnote+:not(.footnote){margin-top:24px}.rst-content .wy-table-responsive.citation:last-child,.rst-content .wy-table-responsive.footnote:last-child{margin-bottom:24px}.rst-content table.docutils th{border-color:#e1e4e5}html.writer-html5 .rst-content table.docutils th{border:1px solid #e1e4e5}html.writer-html5 .rst-content table.docutils td>p,html.writer-html5 .rst-content table.docutils th>p{line-height:1rem;margin-bottom:0;font-size:.9rem}.rst-content table.docutils td .last,.rst-content table.docutils td .last>:last-child{margin-bottom:0}.rst-content table.field-list,.rst-content table.field-list td{border:none}.rst-content table.field-list td p{line-height:inherit}.rst-content table.field-list td>strong{display:inline-block}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left}.rst-content code,.rst-content tt{color:#000;font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;padding:2px 5px}.rst-content code big,.rst-content code em,.rst-content tt big,.rst-content tt em{font-size:100%!important;line-height:normal}.rst-content code.literal,.rst-content tt.literal{color:#e74c3c;white-space:normal}.rst-content code.xref,.rst-content tt.xref,a .rst-content code,a .rst-content tt{font-weight:700;color:#404040;overflow-wrap:normal}.rst-content kbd,.rst-content pre,.rst-content samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace}.rst-content a code,.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:700;margin-bottom:12px}.rst-content dl ol,.rst-content dl p,.rst-content dl table,.rst-content dl ul{margin-bottom:12px}.rst-content dl dd{margin:0 0 12px 24px;line-height:24px}.rst-content dl dd>ol:last-child,.rst-content dl dd>p:last-child,.rst-content dl dd>table:last-child,.rst-content dl dd>ul:last-child{margin-bottom:0}html.writer-html4 .rst-content dl:not(.docutils),html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple){margin-bottom:24px}html.writer-html4 .rst-content dl:not(.docutils)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{display:table;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:3px solid #6ab0de;padding:6px;position:relative}html.writer-html4 .rst-content dl:not(.docutils)>dt:before,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:before{color:#6ab0de}html.writer-html4 .rst-content dl:not(.docutils)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt{margin-bottom:6px;border:none;border-left:3px solid #ccc;background:#f0f0f0;color:#555}html.writer-html4 .rst-content dl:not(.docutils) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) dl:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt .headerlink{color:#404040;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils)>dt:first-child,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple)>dt:first-child{margin-top:0}html.writer-html4 .rst-content dl:not(.docutils) code.descclassname,html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descclassname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{background-color:transparent;border:none;padding:0;font-size:100%!important}html.writer-html4 .rst-content dl:not(.docutils) code.descname,html.writer-html4 .rst-content dl:not(.docutils) tt.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) code.descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) tt.descname{font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .optional,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:700}html.writer-html4 .rst-content dl:not(.docutils) .property,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .property{display:inline-block;padding-right:8px;max-width:100%}html.writer-html4 .rst-content dl:not(.docutils) .k,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .k{font-style:italic}html.writer-html4 .rst-content dl:not(.docutils) .descclassname,html.writer-html4 .rst-content dl:not(.docutils) .descname,html.writer-html4 .rst-content dl:not(.docutils) .sig-name,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descclassname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .descname,html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) .sig-name{font-family:SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;color:#000}.rst-content .viewcode-back,.rst-content .viewcode-link{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right}.rst-content p.rubric{margin-bottom:12px;font-weight:700}.rst-content code.download,.rst-content tt.download{background:inherit;padding:inherit;font-weight:400;font-family:inherit;font-size:inherit;color:inherit;border:inherit;white-space:inherit}.rst-content code.download span:first-child,.rst-content tt.download span:first-child{-webkit-font-smoothing:subpixel-antialiased}.rst-content code.download span:first-child:before,.rst-content tt.download span:first-child:before{margin-right:4px}.rst-content .guilabel,.rst-content .menuselection{font-size:80%;font-weight:700;border-radius:4px;padding:2.4px 6px;margin:auto 2px}.rst-content .guilabel,.rst-content .menuselection{border:1px solid #7fbbe3;background:#e7f2fa}.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>.kbd,.rst-content :not(dl.option-list)>:not(dt):not(kbd):not(.kbd)>kbd{color:inherit;font-size:80%;background-color:#fff;border:1px solid #a6a6a6;border-radius:4px;box-shadow:0 2px grey;padding:2.4px 6px;margin:auto 0}.rst-content .versionmodified{font-style:italic}@media screen and (max-width:480px){.rst-content .sidebar{width:100%;float:none;margin-left:0}}span[id*=MathJax-Span]{color:#404040}.math{text-align:center}@font-face{font-family:Lato;src:url(fonts/lato-normal.woff2?bd03a2cc277bbbc338d464e679fe9942) format("woff2"),url(fonts/lato-normal.woff?27bd77b9162d388cb8d4c4217c7c5e2a) format("woff");font-weight:400;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold.woff2?cccb897485813c7c256901dbca54ecf2) format("woff2"),url(fonts/lato-bold.woff?d878b6c29b10beca227e9eef4246111b) format("woff");font-weight:700;font-style:normal;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-bold-italic.woff2?0b6bb6725576b072c5d0b02ecdd1900d) format("woff2"),url(fonts/lato-bold-italic.woff?9c7e4e9eb485b4a121c760e61bc3707c) format("woff");font-weight:700;font-style:italic;font-display:block}@font-face{font-family:Lato;src:url(fonts/lato-normal-italic.woff2?4eb103b4d12be57cb1d040ed5e162e9d) format("woff2"),url(fonts/lato-normal-italic.woff?f28f2d6482446544ef1ea1ccc6dd5892) format("woff");font-weight:400;font-style:italic;font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:400;src:url(fonts/Roboto-Slab-Regular.woff2?7abf5b8d04d26a2cafea937019bca958) format("woff2"),url(fonts/Roboto-Slab-Regular.woff?c1be9284088d487c5e3ff0a10a92e58c) format("woff");font-display:block}@font-face{font-family:Roboto Slab;font-style:normal;font-weight:700;src:url(fonts/Roboto-Slab-Bold.woff2?9984f4a9bda09be08e83f2506954adbe) format("woff2"),url(fonts/Roboto-Slab-Bold.woff?bed5564a116b05148e3b3bea6fb1162a) format("woff");font-display:block} \ No newline at end of file diff --git a/docs/build/html/_static/sphinx_highlight.js b/docs/build/html/_static/sphinx_highlight.js index aae669d..a74e103 100644 --- a/docs/build/html/_static/sphinx_highlight.js +++ b/docs/build/html/_static/sphinx_highlight.js @@ -1,7 +1,7 @@ /* Highlighting utilities for Sphinx HTML documentation. */ "use strict"; -const SPHINX_HIGHLIGHT_ENABLED = true +const SPHINX_HIGHLIGHT_ENABLED = true; /** * highlight a given string on a node by wrapping it in @@ -13,9 +13,9 @@ const _highlight = (node, addItems, text, className) => { const parent = node.parentNode; const pos = val.toLowerCase().indexOf(text); if ( - pos >= 0 && - !parent.classList.contains(className) && - !parent.classList.contains("nohighlight") + pos >= 0 + && !parent.classList.contains(className) + && !parent.classList.contains("nohighlight") ) { let span; @@ -29,19 +29,18 @@ const _highlight = (node, addItems, text, className) => { } span.appendChild(document.createTextNode(val.substr(pos, text.length))); - parent.insertBefore( - span, - parent.insertBefore( - document.createTextNode(val.substr(pos + text.length)), - node.nextSibling - ) - ); + const rest = document.createTextNode(val.substr(pos + text.length)); + parent.insertBefore(span, parent.insertBefore(rest, node.nextSibling)); node.nodeValue = val.substr(0, pos); + /* There may be more occurrences of search term in this node. So call this + * function recursively on the remaining fragment. + */ + _highlight(rest, addItems, text, className); if (isInSVG) { const rect = document.createElementNS( "http://www.w3.org/2000/svg", - "rect" + "rect", ); const bbox = parent.getBBox(); rect.x.baseVal.value = bbox.x; @@ -60,7 +59,7 @@ const _highlightText = (thisNode, text, className) => { let addItems = []; _highlight(thisNode, addItems, text, className); addItems.forEach((obj) => - obj.parent.insertAdjacentElement("beforebegin", obj.target) + obj.parent.insertAdjacentElement("beforebegin", obj.target), ); }; @@ -68,25 +67,31 @@ const _highlightText = (thisNode, text, className) => { * Small JavaScript module for the documentation. */ const SphinxHighlight = { - /** * highlight the search words provided in localstorage in the text */ highlightSearchWords: () => { - if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight + if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight // get and clear terms from localstorage const url = new URL(window.location); const highlight = - localStorage.getItem("sphinx_highlight_terms") - || url.searchParams.get("highlight") - || ""; - localStorage.removeItem("sphinx_highlight_terms") - url.searchParams.delete("highlight"); - window.history.replaceState({}, "", url); + localStorage.getItem("sphinx_highlight_terms") + || url.searchParams.get("highlight") + || ""; + localStorage.removeItem("sphinx_highlight_terms"); + // Update history only if '?highlight' is present; otherwise it + // clears text fragments (not set in window.location by the browser) + if (url.searchParams.has("highlight")) { + url.searchParams.delete("highlight"); + window.history.replaceState({}, "", url); + } // get individual terms from highlight string - const terms = highlight.toLowerCase().split(/\s+/).filter(x => x); + const terms = highlight + .toLowerCase() + .split(/\s+/) + .filter((x) => x); if (terms.length === 0) return; // nothing to do // There should never be more than one element matching "div.body" @@ -102,11 +107,11 @@ const SphinxHighlight = { document .createRange() .createContextualFragment( - '

' + - '' + - _("Hide Search Matches") + - "

" - ) + '

' + + '' + + _("Hide Search Matches") + + "

", + ), ); }, @@ -120,7 +125,7 @@ const SphinxHighlight = { document .querySelectorAll("span.highlighted") .forEach((el) => el.classList.remove("highlighted")); - localStorage.removeItem("sphinx_highlight_terms") + localStorage.removeItem("sphinx_highlight_terms"); }, initEscapeListener: () => { @@ -129,10 +134,15 @@ const SphinxHighlight = { document.addEventListener("keydown", (event) => { // bail for input elements - if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) + return; // bail with special keys - if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; - if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) + return; + if ( + DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS + && event.key === "Escape" + ) { SphinxHighlight.hideSearchWords(); event.preventDefault(); } @@ -140,5 +150,10 @@ const SphinxHighlight = { }, }; -_ready(SphinxHighlight.highlightSearchWords); -_ready(SphinxHighlight.initEscapeListener); +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000..600ce46 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,53 @@ +.navbar-brand.logo .title.logo__title { + display: none; +} + +.navbar-brand img { + max-height: 2.2rem; + width: auto; +} + +.navbar-brand.logo .title.logo__title { + font-size: 1.05rem; + line-height: 1.15; + white-space: nowrap; +} + +@media (max-width: 1200px) { + .bd-header .navbar-header-items { + min-width: 0; + } + + .bd-header .navbar-header-items__center { + overflow-x: auto; + } + + .bd-header .bd-navbar-elements { + flex-wrap: nowrap; + } +} + +.hero-copy { + font-size: 1.15rem; + line-height: 1.7; + max-width: 56rem; + margin: 0 0 1.5rem 0; +} + +.landing-grid { + margin: 1.2rem 0 2rem 0; +} + +.landing-card { + border-radius: 1rem; + border: 1px solid var(--pst-color-border, #d0d7de); + box-shadow: 0 10px 24px rgba(15, 23, 42, 0.08); +} + +.landing-card .sd-card-title { + font-size: 1.1rem; +} + +[data-theme="dark"] .landing-card { + box-shadow: 0 10px 24px rgba(0, 0, 0, 0.22); +} diff --git a/docs/source/_static/quapy_logo.png b/docs/source/_static/quapy_logo.png new file mode 100644 index 0000000..3798993 Binary files /dev/null and b/docs/source/_static/quapy_logo.png differ diff --git a/docs/source/_static/quapy_logo_dark.png b/docs/source/_static/quapy_logo_dark.png new file mode 100644 index 0000000..18ffbad Binary files /dev/null and b/docs/source/_static/quapy_logo_dark.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 689cc6e..a40a690 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,12 +44,15 @@ extensions = [ 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'myst_parser', + 'sphinx_design', ] autosectionlabel_prefix_document = True source_suffix = ['.rst', '.md'] +myst_enable_extensions = ['colon_fence'] + templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and @@ -61,10 +64,26 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'sphinx_rtd_theme' +#html_theme = 'sphinx_rtd_theme' +html_theme = 'pydata_sphinx_theme' # html_theme = 'furo' # need to be installed: pip install furo (not working...) -# html_static_path = ['_static'] +html_static_path = ['_static'] +html_css_files = ['custom.css'] +html_theme_options = { + 'logo': { + 'image_light': '_static/quapy_logo.png', + 'image_dark': '_static/quapy_logo_dark.png', + }, + 'icon_links': [ + { + 'name': 'GitHub', + 'url': 'https://github.com/HLT-ISTI/QuaPy', + 'icon': 'fa-brands fa-github', + 'type': 'fontawesome', + }, + ], +} # intersphinx configuration intersphinx_mapping = { diff --git a/docs/source/index.md b/docs/source/index.md index d52093e..3a78ba8 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,16 +1,68 @@ ```{toctree} :hidden: -self +Home +manuals +API ``` -# Quickstart +# QuaPy -QuaPy is an open source framework for quantification (a.k.a. supervised prevalence estimation, or learning to quantify) written in Python. +```{div} hero-copy +QuaPy is an open-source Python framework for quantification, also known as +supervised prevalence estimation or learning to quantify. It is designed with +research and experimental analysis in mind, and combines datasets, protocols, +evaluation measures, visualization tools, and a broad collection of +quantification methods in a single workflow. +``` -QuaPy is based on the concept of "data sample", and provides implementations of the most important aspects of the quantification workflow, such as (baseline and advanced) quantification methods, quantification-oriented model selection mechanisms, evaluation measures, and evaluations protocols used for evaluating quantification methods. QuaPy also makes available commonly used datasets, and offers visualization tools for facilitating the analysis and interpretation of the experimental results. +`````{grid} 1 1 2 2 +:gutter: 3 +:class-container: landing-grid -QuaPy is hosted on GitHub at [https://github.com/HLT-ISTI/QuaPy](https://github.com/HLT-ISTI/QuaPy). +````{grid-item-card} Quickstart +:class-card: landing-card +Install QuaPy and run your first quantifier in a few lines of code. ++++ +```{button-link} #installation +:color: primary +Get Started +``` +```` + +````{grid-item-card} Manuals +:class-card: landing-card +Hands-on guides with methodological context, literature pointers, and reproducible workflows. ++++ +```{button-ref} manuals +:ref-type: doc +:color: primary +Open Manuals +``` +```` + +````{grid-item-card} API +:class-card: landing-card +Browse the full reference for `quapy`, including methods, datasets, utilities, and research-oriented extensions. ++++ +```{button-ref} quapy +:ref-type: doc +:color: primary +Browse API +``` +```` + +````{grid-item-card} GitHub +:class-card: landing-card +Explore the source code, open issues, and current development branch activity. ++++ +```{button-link} https://github.com/HLT-ISTI/QuaPy +:color: primary +Open GitHub +``` +```` + +````` ## Installation @@ -18,16 +70,37 @@ QuaPy is hosted on GitHub at [https://github.com/HLT-ISTI/QuaPy](https://github. pip install quapy ``` -## Usage +## Why QuaPy -The following script fetches a dataset of tweets, trains, applies, and evaluates a quantifier based on the *Adjusted Classify & Count* quantification method, using, as the evaluation measure, the *Mean Absolute Error* (MAE) between the predicted and the true class prevalence values of the test set: +QuaPy is built around the concept of a data sample and supports the main tasks +in the quantification workflow: training quantifiers, generating evaluation +samples, measuring quantification error, selecting models under distribution +shift, and visualizing experimental behaviour. The framework is especially +suited for research settings, where one often needs not only implementations, +but also methodological context, literature links, and reproducible evaluation +procedures. + +Some of the main features are: + +* Implementation of many popular quantification methods, including Classify & Count and its variants, + Expectation Maximization, HDy, QuaNet, quantification ensembles, and Bayesian extensions. +* Evaluation protocols for generating test samples under prior probability shift. +* A broad set of quantification-oriented evaluation metrics. +* Ready-to-use textual, numeric, and benchmark competition datasets. +* Method documentation that points back to the relevant literature and original papers. +* Native support for binary and single-label multiclass quantification. +* Visualization tools for analysing predictions, drift, confidence regions, and ternary prevalences. + +## First Example + +The following script fetches a binary dataset, trains an Adjusted Classify & Count quantifier, +and evaluates the resulting prevalence prediction with Mean Absolute Error. ```python import quapy as qp training, test = qp.datasets.fetch_UCIBinaryDataset("yeast").train_test -# create an "Adjusted Classify & Count" quantifier model = qp.method.aggregative.ACC() Xtr, ytr = training.Xy model.fit(Xtr, ytr) @@ -39,43 +112,14 @@ error = qp.error.mae(true_prevalence, estim_prevalence) print(f'Mean Absolute Error (MAE)={error:.3f}') ``` -Quantification is useful in scenarios characterized by prior probability shift. In other words, we would be little interested in estimating the class prevalence values of the test set if we could assume the IID assumption to hold, as this prevalence would be roughly equivalent to the class prevalence of the training set. For this reason, any quantification model should be tested across many samples, even ones characterized by class prevalence values different or very different from those found in the training set. QuaPy implements sampling procedures and evaluation protocols that automate this workflow. See the [](./manuals) for detailed examples. - -## Manuals - -The following manuals illustrate several aspects of QuaPy through examples: - -```{toctree} -:maxdepth: 3 - -manuals -``` - -```{toctree} -:hidden: - -API -``` - -## Features - -* Implementation of many popular quantification methods (Classify-&-Count and its variants, Expectation Maximization, -quantification methods based on structured output learning, HDy, QuaNet, quantification ensembles, among others). -* Versatile functionality for performing evaluation based on sampling generation protocols (e.g., APP, NPP, etc.). -* Implementation of most commonly used evaluation metrics (e.g., AE, RAE, NAE, NRAE, SE, KLD, NKLD, etc.). -* Datasets frequently used in quantification (textual and numeric), including: - * 32 UCI Machine Learning datasets. - * 11 Twitter quantification-by-sentiment datasets. - * 3 product reviews quantification-by-sentiment datasets. - * 4 tasks from LeQua 2022 competition and 4 tasks from LeQua 2024 competition - * IFCB for Plancton quantification -* Native support for binary and single-label multiclass quantification scenarios. -* Model selection functionality that minimizes quantification-oriented loss functions. -* Visualization tools for analysing the experimental results. +Quantification is especially useful when the class prevalence of the test data +may differ from that of the training data. QuaPy implements protocols that make +it easy to evaluate methods across many such shifts. See the [](./manuals) for +worked examples. ## Citing QuaPy -If you find QuaPy useful (and we hope you will), please consider citing the original paper in your research. +If you find QuaPy useful, please consider citing the original paper. ```bibtex @inproceedings{moreo2021quapy, @@ -89,7 +133,8 @@ If you find QuaPy useful (and we hope you will), please consider citing the orig ## Contributing -In case you want to contribute improvements to quapy, please generate pull request to the "devel" branch. +If you want to contribute improvements to QuaPy, please open a pull request +against the `devel` branch. ## Acknowledgments @@ -98,6 +143,6 @@ In case you want to contribute improvements to quapy, please generate pull reque :alt: SoBigData++ ``` -This work has been supported by the QuaDaSh project -_"Finanziato dall’Unione europea---Next Generation EU, +This work has been supported by the QuaDaSh project +_"Finanziato dall'Unione europea---Next Generation EU, Missione 4 Componente 2 CUP B53D23026250001"_. diff --git a/docs/source/manuals.rst b/docs/source/manuals.rst index a426786..97e0628 100644 --- a/docs/source/manuals.rst +++ b/docs/source/manuals.rst @@ -7,7 +7,6 @@ Manuals manuals/datasets manuals/evaluation - manuals/explicit-loss-minimization manuals/methods manuals/model-selection manuals/plotting diff --git a/docs/source/manuals/datasets.md b/docs/source/manuals/datasets.md index b7d8827..50ed9ee 100644 --- a/docs/source/manuals/datasets.md +++ b/docs/source/manuals/datasets.md @@ -412,11 +412,82 @@ ECML-PKDD 2024, Vilnius, Lithuania. ``` +## Image Embedding Datasets + +QuaPy also provides a collection of image datasets in the form of pre-generated +embeddings. +These +embeddings were generated using [this extraction script](https://github.com/pglez82/visiondatasets_quapy) +and are hosted in [Zenodo](https://zenodo.org/records/21131944). + +An example of current public interface is: + +```python +import quapy as qp + +data = qp.datasets.fetch_image_embeddings( + dataset_name='cifar10', + embedding='features', + heldout_only=True, +) +train, test = data.train_test +``` + +The available datasets are in `qp.datasets.IMAGE_DATASETS`, and include 6 datasets: + +* `cifar10`, `cifar100`, and `cifar100coarse`: + [Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, 2009.](https://cave.cs.toronto.edu/kriz/learning-features-2009-TR.pdf) +* `mnist`: + [Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998.](http://yann.lecun.com/exdb/mnist/) +* `fashionmnist`: + [Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017.](https://arxiv.org/abs/1708.07747) +* `svhn`: + [Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y. Ng, et al. Reading digits in natural images with unsupervised feature learning. NIPS Workshop, 2011.](https://static.googleusercontent.com/media/research.google.com/es//pubs/archive/37648.pdf) + + +The available embedding types are in `qp.datasets.IMAGE_EMBEDDINGS`, and include: + +* `features` are the penultimate-layer representations +* `logits` are the pre-activation outputs of the neural model +* `predictions` are the post-softmax posterior probabilities + +The datasets correspond to frozen neural representations extracted from models +trained on image classification tasks. QuaPy downloads them automatically on +first use and stores them locally for fast reuse. + +Each dataset is internally organised into three splits: `train`, `val`, and +`test`. The `train` split was used to train the neural model that produced the +embeddings, while `val` and `test` were not seen during neural training. +For this reason, the default setting indicates `heldout_only=True`, meaning +that the returned dataset will take the validation partition as the training +set, and the test partition as the test set. +This is often the most convenient choice for quantification experiments, since +it avoids training quantifiers on examples that were already used to train the +embedding model. + +If instead you want to use all the available non-test data, you can set `heldout_only=False`, +in which case, the returned training set is the union of the original neural +training split and the validation split. + +Some statistics are shown in the following table: + +| Dataset | backbone | classes | neural network train size | validation size | test size | feature dim | logit dim | prediction dim | type | +|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|---| +| cifar100 | resnet18 | 100 | 35000 | 15000 | 10000 | 512 | 100 | 100 | dense | +| cifar10 | resnet18 | 10 | 35000 | 15000 | 10000 | 512 | 10 | 10 | dense | +| cifar100coarse | resnet18 | 20 | 35000 | 15000 | 10000 | 512 | 20 | 20 | dense | +| mnist | basiccnn | 10 |42000 | 18000 | 10000 | 128 | 10 | 10 | dense | +| fashionmnist | basiccnn | 10 | 42000 | 18000 | 10000 | 128 | 10 | 10 | dense | +| svhn | resnet18 | 10 | 51280 | 21977 | 26032 | 512 | 10 | 10 | dense | + + + + ## IFCB Plankton dataset -IFCB is a dataset of plankton species in water samples hosted in `Zenodo `_. -This dataset is based on the data available publicly at `WHOI-Plankton repo `_ -and in the scripts for the processing are available at `P. González's repo `_. +IFCB is a dataset of plankton species in water samples hosted in [Zenodo](https://zenodo.org/records/10036244). +This dataset is based on the data available publicly at [WHOI-Plankton repo](https://github.com/hsosik/WHOI-Plankton) +and the scripts for the processing are available at [P. González's repo](https://github.com/pglez82/IFCB_Zenodo). This dataset comes with precomputed features for testing quantification algorithms. diff --git a/docs/source/manuals/evaluation.md b/docs/source/manuals/evaluation.md index aba7068..f0af07e 100644 --- a/docs/source/manuals/evaluation.md +++ b/docs/source/manuals/evaluation.md @@ -65,6 +65,116 @@ error_function = qp.error.from_name('mse') error = error_function(true_prev, estim_prev) ``` +The main quantification measures currently available in `qp.error` are the +following. As a rule of thumb, names starting with `m` indicate the mean value +across many sample pairs, while the corresponding unprefixed function returns +the sample-wise quantity. Let `p` denote the true prevalence vector, +`\hat{p}` the predicted prevalence vector, `\mathcal{Y}` the set of classes, +and `p^{tr}` the training prevalence vector. + +### Prevalence-vector measures + +Absolute error and its mean version: + +```{math} +AE(p,\hat{p}) = \frac{1}{|\mathcal{Y}|}\sum_{y \in \mathcal{Y}} |\hat{p}(y)-p(y)| +``` + +Implemented as `ae` and `mae`. + +Normalized absolute error and its mean version: + +```{math} +NAE(p,\hat{p}) = \frac{AE(p,\hat{p})}{z_{AE}},\qquad +z_{AE}=\frac{2(1-\min_{y \in \mathcal{Y}} p(y))}{|\mathcal{Y}|} +``` + +Implemented as `nae` and `mnae`. + +Squared error and its mean version: + +```{math} +SE(p,\hat{p}) = \frac{1}{|\mathcal{Y}|}\sum_{y \in \mathcal{Y}} (\hat{p}(y)-p(y))^2 +``` + +Implemented as `se` and `mse`. + +Relative absolute error and its mean version: + +```{math} +RAE(p,\hat{p}) = \frac{1}{|\mathcal{Y}|}\sum_{y \in \mathcal{Y}}\frac{|\hat{p}(y)-p(y)|}{p(y)} +``` + +Implemented as `rae` and `mrae`. + +Normalized relative absolute error and its mean version: + +```{math} +NRAE(p,\hat{p}) = \frac{RAE(p,\hat{p})}{z_{RAE}},\qquad +z_{RAE}=\frac{|\mathcal{Y}|-1+\frac{1-\min_{y \in \mathcal{Y}} p(y)}{\min_{y \in \mathcal{Y}} p(y)}}{|\mathcal{Y}|} +``` + +Implemented as `nrae` and `mnrae`. + +Kullback-Leibler divergence and its mean version: + +```{math} +KLD(p,\hat{p}) = \sum_{y \in \mathcal{Y}} p(y)\log\frac{p(y)}{\hat{p}(y)} +``` + +Implemented as `kld` and `mkld`. + +Normalized Kullback-Leibler divergence and its mean version: + +```{math} +NKLD(p,\hat{p}) = 2\frac{e^{KLD(p,\hat{p})}}{e^{KLD(p,\hat{p})}+1}-1 +``` + +Implemented as `nkld` and `mnkld`. + +Squared ratio error and its mean version: + +```{math} +SRE(p,\hat{p},p^{tr}) = \frac{1}{|\mathcal{Y}|}\sum_{i \in \mathcal{Y}} (w_i-\hat{w}_i)^2,\qquad +w_i=\frac{p_i}{p^{tr}_i} +``` + +Implemented as `sre` and `msre`. + +The Aitchison Quantification Error (AQE) and its mean version (MAQE) are implemented as `aqe` and `maqe` using the +Aitchison Distance (available in `qp.functional.AitchisonDistance`, here denoted `d_A`): + +```{math} +d_A(p,\hat{p}) = \|\mathrm{clr}(p)-\mathrm{clr}(\hat{p})\|_2 +``` + +### Additional measures + +Match distance computes the cumulative-distribution discrepancy under the +assumption that moving mass from class `i` to class `i+1` has unit cost: + +```{math} +MD(p,\hat{p}) = \sum_{i=1}^{|\mathcal{Y}|-1} \left|\sum_{j=1}^{i} p_j - \sum_{j=1}^{i} \hat{p}_j\right| +``` + +Implemented as `md`. Its normalized variant `nmd` rescales this quantity by +`1/(|\mathcal{Y}|-1)`. + +For binary quantification, QuaPy also provides the signed bias of the positive +class and its mean value: + +```{math} +bias(p,\hat{p}) = \hat{p}_1 - p_1 +``` + +Implemented as `bias_binary` and `mean_bias_binary`. + +### Classification measures + +The same module also exposes two classification-oriented error measures, which +can occasionally be useful for diagnostics: `acce` (accuracy error, i.e., +`1-accuracy`) and `f1e` (macro-`F_1` error, i.e., `1-F_1^M`). + ## Evaluation Protocols An _evaluation protocol_ is an evaluation procedure that uses diff --git a/docs/source/manuals/explicit-loss-minimization.md b/docs/source/manuals/explicit-loss-minimization.md deleted file mode 100644 index f80c434..0000000 --- a/docs/source/manuals/explicit-loss-minimization.md +++ /dev/null @@ -1,26 +0,0 @@ -# Explicit Loss Minimization - -QuaPy makes available several Explicit Loss Minimization (ELM) methods, including -SVM(Q), SVM(KLD), SVM(NKLD), SVM(AE), or SVM(RAE). -These methods require to first download the -[svmperf](http://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html) -package, apply the patch -[svm-perf-quantification-ext.patch](https://github.com/HLT-ISTI/QuaPy/blob/master/svm-perf-quantification-ext.patch), and compile the sources. -The script [prepare_svmperf.sh](https://github.com/HLT-ISTI/QuaPy/blob/master/prepare_svmperf.sh) does all the job. Simply run: - -``` -./prepare_svmperf.sh -``` - -The resulting directory `svm_perf_quantification/` contains the -patched version of _svmperf_ with quantification-oriented losses. - -The [svm-perf-quantification-ext.patch](https://github.com/HLT-ISTI/QuaPy/blob/master/prepare_svmperf.sh) is an extension of the patch made available by -[Esuli et al. 2015](https://dl.acm.org/doi/abs/10.1145/2700406?casa_token=8D2fHsGCVn0AAAAA:ZfThYOvrzWxMGfZYlQW_y8Cagg-o_l6X_PcF09mdETQ4Tu7jK98mxFbGSXp9ZSO14JkUIYuDGFG0) -that allows SVMperf to optimize for -the _Q_ measure as proposed by [Barranquero et al. 2015](https://www.sciencedirect.com/science/article/abs/pii/S003132031400291X) -and for the _KLD_ and _NKLD_ measures as proposed by [Esuli et al. 2015](https://dl.acm.org/doi/abs/10.1145/2700406?casa_token=8D2fHsGCVn0AAAAA:ZfThYOvrzWxMGfZYlQW_y8Cagg-o_l6X_PcF09mdETQ4Tu7jK98mxFbGSXp9ZSO14JkUIYuDGFG0). -This patch extends the above one by also allowing SVMperf to optimize for -_AE_ and _RAE_. -See the [](./methods) manual for more details and code examples. - diff --git a/docs/source/manuals/methods.md b/docs/source/manuals/methods.md index 47b7cad..4d7d75e 100644 --- a/docs/source/manuals/methods.md +++ b/docs/source/manuals/methods.md @@ -2,11 +2,17 @@ Quantification methods can be categorized as belonging to `aggregative`, `non-aggregative`, and `meta-learning` groups. -Most methods included in QuaPy at the moment are of type `aggregative` -(though we plan to add many more methods in the near future), i.e., -are methods characterized by the fact that -quantification is performed as an aggregation function of the individual -products of classification. +`Aggregative` quantifiers rely on a surrogate classifier as an intermediate +step, and devise different aggregation functions over the classifier outputs. +By contrast, `non-aggregative` methods perform quantification without requiring +an underlying classifier. +`Meta-learning` refers to quantification methods that are constructed over simpler +quantification methods, and implement high-level orchestration functions. + +Beyond these three traditional categories of methods, we here present an additional, +orthogonal one: `Bayesian quantifiers`, i.e., quantification methods that do not simply +return point-estimates of class prevalence, but are also able to provide a measure of +uncertaintly around them. Any quantifier in QuaPy shoud extend the class `BaseQuantifier`, and implement some abstract methods: @@ -112,11 +118,15 @@ in evaluation. QuaPy implements the four CC variants, i.e.: * _CC_ (Classify & Count), the simplest aggregative quantifier; one that - simply relies on the label predictions of a classifier to deliver class estimates. -* _ACC_ (Adjusted Classify & Count), the adjusted variant of CC. + classifies all instances and computes the prevalence of the predicted labels. + This baseline is discussed, among others, in [Forman (2008)](https://link.springer.com/article/10.1007/s10618-008-0097-y). +* _ACC_ (Adjusted Classify & Count), the adjusted variant of CC, originally + proposed in [Forman (2008)](https://link.springer.com/article/10.1007/s10618-008-0097-y). * _PCC_ (Probabilistic Classify & Count), the probabilistic variant of CC that -relies on the soft estimations (or posterior probabilities) returned by a (probabilistic) classifier. -* _PACC_ (Probabilistic Adjusted Classify & Count), the adjusted variant of PCC. + relies on the posterior probabilities returned by a probabilistic classifier, + introduced in [Bella et al. (2010)](https://ieeexplore.ieee.org/abstract/document/5694031). +* _PACC_ (Probabilistic Adjusted Classify & Count), the adjusted variant of PCC, + also introduced in [Bella et al. (2010)](https://ieeexplore.ieee.org/abstract/document/5694031). The following code serves as a complete example using CC equipped with a SVM as the classifier: @@ -136,7 +146,7 @@ svm = LinearSVC() # (an alias is available in qp.method.aggregative.ClassifyAndCount) model = qp.method.aggregative.CC(svm) model.fit(Xtr, ytr) -estim_prevalence = model.predict(test.instances) +estim_prevalence = model.predict(test.X) ``` The same code could be used to instantiate an ACC, by simply replacing @@ -184,6 +194,10 @@ will be raised otherwise. Lastly, everything we said about ACC and PCC applies to PACC as well. +A Bayesian counterpart of the ACC family is also available; see the +{ref}`Bayesian Quantification Methods section ` +for `BayesianCC`. + _New in v0.1.9_: quantifiers ACC and PACC now have three additional arguments: `method`, `solver` and `norm`: * Argument `method` specifies how to solve, for `p`, the linear system `q = Mp` (where `q` is the unadjusted counts for the @@ -213,24 +227,23 @@ Options are: * `"condsoftmax"` applies softmax normalization only if the prevalence vector lies outside of the probability simplex. -#### BayesianCC +### Threshold Optimization methods -The `BayesianCC` is a variant of ACC introduced in -[Ziegler, A. and Czyż, P. "Bayesian quantification with black-box estimators", arXiv (2023)](https://arxiv.org/abs/2302.09159), -which models the probabilities `q = Mp` using latent random variables with weak Bayesian priors, rather than -plug-in probability estimates. In particular, it uses Markov Chain Monte Carlo sampling to find the values of -`p` compatible with the observed quantities. -The `aggregate` method returns the posterior mean and the `get_prevalence_samples` method can be used to find -uncertainty around `p` estimates (conditional on the observed data and the trained classifier) -and is suitable for problems in which the `q = Mp` matrix is nearly non-invertible. +QuaPy implements Forman's threshold optimization methods; +see, e.g., [(Forman 2006)](https://dl.acm.org/doi/abs/10.1145/1150402.1150423) +and [(Forman 2008)](https://link.springer.com/article/10.1007/s10618-008-0097-y). +These include: `T50`, `MAX`, `X`, Median Sweep (`MS`), and its variant `MS2`. -Note that this quantification method requires `val_split` to be a `float` and installation of additional dependencies (`$ pip install quapy[bayes]`) needed to run Markov chain Monte Carlo sampling. Markov Chain Monte Carlo is is slower than matrix inversion methods, but is guaranteed to sample proper probability vectors, so no clipping strategies are required. -An example presenting how to run the method and use posterior samples is available in `examples/bayesian_quantification.py`. +These methods are binary-only and implement different heuristics for +improving the stability of the denominator of the ACC adjustment (`tpr-fpr`). +The methods are called "threshold" since said heuristics have to do +with different choices of the underlying classifier's threshold. -### Expectation Maximization (EMQ) -The Expectation Maximization Quantifier (EMQ), also known as -the SLD, is available at `qp.method.aggregative.EMQ` or via the +### Expectation Maximization (EMQ) / Maximum Likelihood for Label Shift (MLLS) + +The Expectation Maximization Quantifier (EMQ) (also known as +SLD after the name of the proponets, or Maximum Likelihood for Label Shift, MLLS) , is available at `qp.method.aggregative.EMQ` or via the alias `qp.method.aggregative.ExpectationMaximizationQuantifier`. The method is described in: @@ -274,8 +287,57 @@ or Temperature Scaling (`ts`); default is `None` (no calibration). You can use the class method `EMQ_BCTS` to effortlessly instantiate EMQ with the best performing heuristics found by [Alexandari et al. (2020)](http://proceedings.mlr.press/v119/alexandari20a.html). See the API documentation for further details. +For a Bayesian label-shift counterpart based on the same general family of ideas, +see the {ref}`Bayesian Quantification Methods section ` +for `BayesianMAPLS`. -### Hellinger Distance y (HDy) +### Regularized Learning under Label Shift (RLLS) + +`RLLS` is available at `qp.method.aggregative.RLLS` and ports the regularized +importance-weight estimation procedure of +[Azizzadenesheli, K., Liu, A., Yang, F., and Anandkumar, A. (2019). Regularized +Learning for Domain Adaptation under Label Shifts. +ICLR 2019](https://arxiv.org/abs/1903.09734) to QuaPy's aggregative interface. +The method estimates the label-shift importance weights `w = q(y)/p(y)` from +the classifier's validation posteriors (or, in `mode='hard'`, its argmax +predictions) and the corresponding source labels, regularizing the estimation +by an amount controlled by `alpha` (scaled by a finite-sample confidence term +governed by `delta`). The resulting weights are then used to rescale the +training prevalence into the target prevalence estimate. + +Like ACC and PACC, RLLS requires validation predictions and therefore expects +`val_split` to be set (as an integer for k-fold cross-validation, a float for +a held-out split, or an explicit `(X, y)` tuple) whenever `fit_classifier=True`. +This method relies on the optional `cvxpy` dependency, which must be +installed separately (`$ pip install cvxpy`). + +```python +import quapy as qp +from quapy.method.aggregative import RLLS +from sklearn.linear_model import LogisticRegression + +train, test = qp.datasets.fetch_UCIBinaryDataset('haberman').train_test + +model = RLLS(LogisticRegression(max_iter=2000), val_split=5) +model.fit(*train.Xy) +estim_prevalence = model.predict(test.X) +``` + +### Distribution Matching + +Distribution Matching (DM) methods search for the mixture parameter (the sought class prevalence values) +yielding the mixture between the class-wise representations that best matches the test distribution. +Different criteria for deciding how this matching is assessed, and different ways for modelling the +distributions give rise to different instantiations of DM methods. + +The following methods are here discussed because they rely on a surrogate classifier for representing +the distributions, albeit different non-aggregative variants of them do often exist. Aside from this, +the formulation of DM methods is flexible enough as to accomodate methods that were proposed under a different +framework; examples include ACC and PACC. + +See the frameworks by [Firat](https://arxiv.org/abs/1606.00868), [Bunse](https://dl.gi.de/items/5a61f30f-6c84-4165-bd92-9098bd9e91aa), [Garg et al.](https://dl.acm.org/doi/10.5555/3495724.3496001), or [Dussap](https://theses.hal.science/tel-04931123), for more details. + +#### Hellinger Distance y (HDy) Implementation of the method based on the Hellinger Distance y (HDy) proposed by [González-Castro, V., Alaiz-Rodríguez, R., and Alegre, E. (2013). Class distribution @@ -310,31 +372,94 @@ model.fit(*dataset.training.Xy) estim_prevalence = model.predict(dataset.test.X) ``` -QuaPy also provides an implementation of the generalized -"Distribution Matching" approaches for multiclass, inspired by the framework -of [Firat (2016)](https://arxiv.org/abs/1606.00868). One can instantiate -a variant of HDy for multiclass quantification as follows: +#### Generalized Distribution Matching y (DMy) + +QuaPy also provides a generalized posterior-space distribution-matching +quantifier for binary or multiclass problems, implemented as +`qp.method.aggregative.DMy`. This class follows the generic distribution +matching view discussed by [Firat (2016)](https://arxiv.org/abs/1606.00868): +it represents class-conditional posterior distributions by histograms and then +searches for the prevalence vector whose mixture best matches the test +distribution. + +`DMy` is intentionally flexible and exposes three main design choices: the +number of histogram bins (`nbins`), the divergence to minimize (`divergence`, +e.g., `'HD'` or `'topsoe'`), and whether to match PDFs or CDFs (`cdf`). The +optimization routine can also be selected through `search`; the default +`'optim_minimize'` works for multiclass problems, while `'linear_search'` and +`'ternary_search'` are binary-only. A multiclass HDy-like instance can be +obtained as: ```python -mutliclassHDy = qp.method.aggregative.DMy(classifier=LogisticRegression(), divergence='HD', cdf=False) -``` +multiclass_hdy = qp.method.aggregative.DMy( + classifier=LogisticRegression(), + divergence='HD', + cdf=False, +) +``` -QuaPy also provides an implementation of the "DyS" -framework proposed by [Maletzke et al (2020)](https://ojs.aaai.org/index.php/AAAI/article/view/4376) -and the "SMM" method proposed by [Hassan et al (2019)](https://ieeexplore.ieee.org/document/9260028) -(thanks to _Pablo González_ for the contributions!) +#### DyS -### Threshold Optimization methods +QuaPy implements the binary `DyS` framework proposed by +[Maletzke et al. (2020)](https://ojs.aaai.org/index.php/AAAI/article/view/4376) +as `qp.method.aggregative.DyS`. Conceptually, `DyS` can be seen as a +generalization of HDy in which the prevalence is found by ternary search over a +distribution-matching objective. In QuaPy, the user can select the number of +histogram bins (`n_bins`), the divergence (`divergence`), and the optimization +tolerance (`tol`). -QuaPy implements Forman's threshold optimization methods; -see, e.g., [(Forman 2006)](https://dl.acm.org/doi/abs/10.1145/1150402.1150423) -and [(Forman 2008)](https://link.springer.com/article/10.1007/s10618-008-0097-y). -These include: `T50`, `MAX`, `X`, Median Sweep (`MS`), and its variant `MS2`. +#### Energy Distance y (EDy) + +QuaPy also adapts `EDy` from [quantificationlib](https://github.com/AICGijon/quantificationlib), +which is available as `qp.method.aggregative.EDy`. + +This +method replaces histogram matching with an energy-distance formulation defined +directly on posterior-probability vectors and solves the resulting optimization +problem by quadratic programming. The method is proposed in +[Castaño et al.'s (2024)](https://ieeexplore.ieee.org/document/9791435/) paper. + +In QuaPy, `EDy` works for binary and +multiclass problems and lets the user choose the pairwise distance through the +`distance` parameter (`'manhattan'`, `'euclidean'`, or a custom callable). +Because the optimization relies on `quadprog`, this method requires the +optional dependency `pip install quadprog`. + +#### SMM + +QuaPy also includes the binary `SMM` method of +[Hassan et al. (2019)](https://ieeexplore.ieee.org/document/9260028), +available as `qp.method.aggregative.SMM`. This is a very lightweight +distribution-matching variant in which the posterior representation is reduced +to class-wise means rather than full histograms, making it conceptually close +to PACC. + + +#### Kernel Density Estimation methods (KDEy) + +QuaPy provides implementations for the three variants +of KDE-based methods proposed in +_[Moreo, A., González, P. and del Coz, J.J.. +Kernel Density Estimation for Multiclass Quantification. +Machine Learning. Vol 114 (92), 2025](https://link.springer.com/article/10.1007/s10994-024-06726-5)_ +(a [preprint](https://arxiv.org/abs/2401.00490) is available online). +The variants differ in the divergence metric to be minimized: + +- KDEy-HD: minimizes the (squared) Hellinger Distance and solves the problem via a Monte Carlo approach +- KDEy-CS: minimizes the Cauchy-Schwarz divergence and solves the problem via a closed-form solution +- KDEy-ML: minimizes the Kullback-Leibler divergence and solves the problem via maximum-likelihood + +These methods are specifically devised for multiclass problems (although they can tackle +binary problems too). + +All KDE-based methods depend on the hyperparameter `bandwidth` of the kernel. Typical values +that can be explored in model selection range in [0.01, 0.25]. Previous experiments reveal the methods' performance +varies smoothly at small variations of this hyperparameter. + +A Bayesian counterpart is available as well; see the +{ref}`Bayesian Quantification Methods section ` +for `BayesianKDEy`. -These methods are binary-only and implement different heuristics for -improving the stability of the denominator of the ACC adjustment (`tpr-fpr`). -The methods are called "threshold" since said heuristics have to do -with different choices of the underlying classifier's threshold. ### Explicit Loss Minimization @@ -363,84 +488,210 @@ the last two methods (SVM(AE) and SVM(RAE)) have been implemented in QuaPy in order to make available ELM variants for what nowadays are considered the most well-behaved evaluation metrics in quantification. -In order to make these models work, you would need to run the script -`prepare_svmperf.sh` (distributed along with QuaPy) that -downloads `SVMperf`' source code, applies a patch that -implements the quantification oriented losses, and compiles the -sources. +#### Installing the SVMperf backend -If you want to add any custom loss, you would need to modify -the source code of `SVMperf` in order to implement it, and -assign a valid loss code to it. Then you must re-compile -the whole thing and instantiate the quantifier in QuaPy -as follows: +These methods rely on Joachim's [SVMperf](https://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html), +patched with quantification-oriented losses. QuaPy provides the script +[`prepare_svmperf.sh`](https://github.com/HLT-ISTI/QuaPy/blob/master/prepare_svmperf.sh), +which downloads the original sources, applies the patch, and compiles the +resulting binary. In practice, this amounts to running: -```python -# you can either set the path to your custom svm_perf_quantification implementation -# in the environment variable, or as an argument to the constructor of ELM -qp.environ['SVMPERF_HOME'] = './path/to/svm_perf_quantification' - -# assign an alias to your custom loss and the id you have assigned to it -svmperf = qp.classification.svmperf.SVMperf -svmperf.valid_losses['mycustomloss'] = 28 - -# instantiate the ELM method indicating the loss -model = qp.method.aggregative.ELM(loss='mycustomloss') +```sh +./prepare_svmperf.sh ``` -All ELM are binary quantifiers since they rely on `SVMperf`, that -currently supports only binary classification. -ELM variants (any binary quantifier in general) can be extended -to operate in single-label scenarios trivially by adopting a -"one-vs-all" strategy (as, e.g., in -[_Gao, W. and Sebastiani, F. (2016). From classification to quantification in tweet sentiment -analysis. Social Network Analysis and Mining, 6(19):1–22_](https://link.springer.com/article/10.1007/s13278-016-0327-z)). -In QuaPy this is possible by using the `OneVsAll` class. +This creates a directory `svm_perf_quantification/`. Once this is available, +you can point QuaPy to it with: -There are two ways for instantiating this class, `OneVsAllGeneric` that works for -any quantifier, and `OneVsAllAggregative` that is optimized for aggregative quantifiers. -In general, you can simply use the `newOneVsAll` function and QuaPy will choose -the more convenient of the two. +```python +qp.environ['SVMPERF_HOME'] = './svm_perf_quantification' +``` + +The patch extends the one originally released for +[Esuli and Sebastiani (2015)](https://dl.acm.org/doi/abs/10.1145/2700406) +and also covers the `Q`, `AE`, and `RAE` losses used by QuaPy's ELM wrappers. + +All ELM methods are binary because `SVMperf` itself is binary. They can still +be wrapped in a one-vs-all scheme for single-label multiclass problems, though +this strategy is generally considered inappropriate under prior probability +shift. See the examples on +[explicit loss minimization](https://github.com/HLT-ISTI/QuaPy/blob/devel/examples/17.explicit_loss_minimization.py) +and on +[one versus all quantification](https://github.com/HLT-ISTI/QuaPy/blob/devel/examples/10.one_vs_all.py) +for minimal working code. + +## Non-Aggregative Methods + +Non-aggregative methods are quantifiers that do not follow the two-step +(classify, then aggregate) pattern described above for aggregative methods. +These methods are implemented in the `qp.method.non_aggregative` module and +extend `BaseQuantifier` directly, implementing `fit` and `predict` on their own terms. + +### Maximum Likelihood Prevalence Estimation (MLPE) + +`MaximumLikelihoodPrevalenceEstimation` (MLPE) is a lazy baseline quantifier +that assumes the IID assumption holds, i.e., that there is no prior probability +shift between the training and the test distributions. Its `fit` method simply +computes and stores the training prevalence, and its `predict` method returns +that same training prevalence for any test sample, irrespective of the sample +itself. MLPE is considered a lower-bound quantifier: any quantification method +worth using should outperform it. ```python import quapy as qp -from quapy.method.aggregative import SVMQ +from quapy.method.non_aggregative import MaximumLikelihoodPrevalenceEstimation -# load a single-label dataset (this one contains 3 classes) -dataset = qp.datasets.fetch_twitter('hcr', pickle=True) +dataset = qp.datasets.fetch_UCIBinaryDataset('haberman') +train, test = dataset.train_test -# let qp know where svmperf is -qp.environ['SVMPERF_HOME'] = '../svm_perf_quantification' - -model = newOneVsAll(SVMQ(), n_jobs=-1) # run them on parallel -model.fit(dataset.training) -estim_prevalence = model.predict(dataset.test.instances) +model = MaximumLikelihoodPrevalenceEstimation() +model.fit(*train.Xy) +estim_prevalence = model.predict(test.X) # always equals train.prevalence() ``` -Check the examples on [explicit loss minimization](https://github.com/HLT-ISTI/QuaPy/blob/devel/examples/17.explicit_loss_minimization.py) -and on [one versus all quantification](https://github.com/HLT-ISTI/QuaPy/blob/devel/examples/10.one_vs_all.py) for more details. -**Note** that the _one versus all_ approach is considered inappropriate under prior probability shift, though. +### Distribution Matching x (DMx) and Hellinger Distance x (HDx) -### Kernel Density Estimation methods (KDEy) +`DMx` is the covariate-space counterpart of the `DMy` distribution-matching +quantifier described in {ref}`the Hellinger Distance y (HDy) section `: +instead of matching distributions built from the classifier's predictions, `DMx` matches +distributions built directly from the (discretized) feature space, and thus +requires no classifier at all. For each class, `DMx` builds one histogram per +feature from the training instances of that class; at prediction time, it +searches for the mixture of these class-conditional histograms that best +matches the (also histogram-based) representation of the test sample, in +terms of a chosen divergence. -QuaPy provides implementations for the three variants -of KDE-based methods proposed in -_[Moreo, A., González, P. and del Coz, J.J.. -Kernel Density Estimation for Multiclass Quantification. -Machine Learning. Vol 114 (92), 2025](https://link.springer.com/article/10.1007/s10994-024-06726-5)_ -(a [preprint](https://arxiv.org/abs/2401.00490) is available online). -The variants differ in the divergence metric to be minimized: +`DMx` accepts the following hyperparameters in its constructor: -- KDEy-HD: minimizes the (squared) Hellinger Distance and solves the problem via a Monte Carlo approach -- KDEy-CS: minimizes the Cauchy-Schwarz divergence and solves the problem via a closed-form solution -- KDEy-ML: minimizes the Kullback-Leibler divergence and solves the problem via maximum-likelihood +* `nbins`: the number of bins used to discretize each feature (default 8) +* `divergence`: a string ("HD" for Hellinger Distance, or "topsoe") or a + callable taking two histograms and returning a divergence value (default "HD") +* `cdf`: whether to match cumulative distributions (CDFs) instead of the + histograms (PDFs) themselves (default False) +* `search`: the strategy used for finding the optimal prevalence; valid + options are `optim_minimize` (default, works for binary and multiclass + problems), `linear_search`, and `ternary_search` (these last two are + binary-only) +* `n_jobs`: number of parallel workers (default None) -These methods are specifically devised for multiclass problems (although they can tackle -binary problems too). +`DMx` also offers the class method `DMx.HDx` (aliased as +`qp.method.non_aggregative.HDx`, and also as `HellingerDistanceX`) that +reproduces the original Hellinger Distance x (HDx) method proposed by +[González-Castro, Alaiz-Rodríguez, and Alegre (2013)](https://www.sciencedirect.com/science/article/pii/S0020025512004069), +the same paper that introduced HDy. HDx is a binary-only method that computes +the matching for `nbins` ranging over `[10, 20, ..., 110]` (via a +`MedianEstimator`, taking the median of the resulting estimates) and searches +for the best prevalence via a linear search stepping by 0.01, rather than via +the `optim_minimize` search used by `DMx` by default. -All KDE-based methods depend on the hyperparameter `bandwidth` of the kernel. Typical values -that can be explored in model selection range in [0.01, 0.25]. Previous experiments reveal the methods' performance -varies smoothly at small variations of this hyperparameter. +The following code, adapted from the example comparing HDy and HDx +(`examples/11.comparing_HDy_HDx.py`), shows the two methods side-by-side: + +```python +from sklearn.linear_model import LogisticRegression +import quapy as qp +from quapy.method.aggregative import HDy +from quapy.method.non_aggregative import DMx + +train, test = qp.datasets.fetch_UCIBinaryDataset('haberman').train_test +Xtr, ytr = train.Xy + +hdy = HDy(LogisticRegression()).fit(Xtr, ytr) +estim_prevalence_hdy = hdy.predict(test.X) + +hdx = DMx.HDx(n_jobs=-1).fit(Xtr, ytr) +estim_prevalence_hdx = hdx.predict(test.X) +``` + +Note that, unlike HDy, HDx requires no classifier whatsoever, since it +operates directly on the covariates. + +### Energy Distance x (EDx) + +QuaPy also provides `qp.method.non_aggregative.EDx`, which is the +feature-space counterpart of `EDy`: it keeps the same energy-distance +formulation and quadratic-program solver, but applies them directly to the raw +instances instead of first projecting them onto posterior probabilities through +a classifier. In this sense, `EDx` is to `EDy` what `DMx` is to `DMy`. + +`EDx` works for binary and multiclass problems, accepts the same `distance` +options as `EDy` (`'manhattan'`, `'euclidean'`, or a custom callable), and +requires the optional dependency `pip install quadprog`. + +### ReadMe + +`ReadMe` is a non-aggregative quantification method proposed by +[Hopkins, D. and King, G. (2007). A method of automated nonparametric content +analysis for social science. American Journal of Political Science, +54(1):229-247.](https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1540-5907.2009.00428.x) +The method estimates `Q(Y=i)` directly from `Q(X) = sum_i Q(X|Y=i) Q(Y=i)` by +solving a (constrained) least-squares regression, thus avoiding the cost of +estimating posterior probabilities `Q(Y=i|X)` altogether. + +Since `Q(X)` and `Q(X|Y=i)` can be of very high dimension for realistic +feature spaces, ReadMe renders the problem tractable by performing bagging in +the feature space: many small random subsets of features (of size +`bagging_range`) are drawn, the least-squares problem is solved on each +subset, and the resulting estimates are averaged. ReadMe additionally +combines this bagging procedure with bootstrap resampling of the training +instances in order to derive confidence regions around the point estimate; +accordingly, `ReadMe` implements the `WithConfidenceABC` interface (see the +{ref}`confidence regions section `), +and exposes a `predict_conf` method in addition to `predict`. + +`ReadMe` accepts the following hyperparameters: + +* `prob_model`: either `"full"` (default), the original Hopkins and King + formulation, in which `Q(X)` and `Q(X|Y)` are modelled empirically and thus + require the feature matrix `X` to be binary (e.g., term presence/absence); + or `"naive"`, a much faster approximation that models `Q(X)` and `Q(X|Y)` as + multinomial (bag-of-words) distributions, and that supports much larger + values of `bagging_range` +* `bootstrap_trials`: number of bootstrap resamplings of the training data + used for deriving the confidence region (default 300) +* `bagging_trials`: number of bagging trials, i.e., random feature subsets, + averaged for each point estimate (default 300) +* `bagging_range`: number of features kept in each bagging trial (default 15); + note that, when `prob_model="full"`, this value should typically be kept + small (the authors advise against values above 25) since the empirical + distribution requires enumerating `2^bagging_range` possible feature + configurations +* `confidence_level`: the confidence level for the confidence region + (default 0.95) +* `region`: the type of confidence region to construct, one of `"intervals"` + (default), `"ellipse"`, `"ellipse-clr"`, or `"ellipse-ilr"` (see the + {ref}`confidence regions section ` + for details) +* `bonferroni`: whether to apply Bonferroni correction when `region="intervals"` + (default `False`); this parameter has no effect for ellipse-based regions +* `random_state`: an int for replicability, or `None` (default) +* `verbose`: whether to display progress information (default False) + +The following minimal example, adapted from +`examples/18.ReadMe_for_text_analysis.py`, shows ReadMe applied to a binary +bag-of-words text quantification problem: + +```python +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.pipeline import Pipeline +import quapy as qp +from quapy.method.non_aggregative import ReadMe + +reviews = qp.datasets.fetch_reviews('imdb').reduce(n_train=1000, random_state=0) + +# ReadMe's "full" model requires a binary feature matrix +encode_0_1 = Pipeline([('0_1_terms', CountVectorizer(min_df=5, binary=True))]) +train, test = qp.data.preprocessing.instance_transformation(reviews, encode_0_1, inplace=True).train_test + +model = ReadMe(prob_model='full', bootstrap_trials=100, bagging_trials=100, bagging_range=20, random_state=0) +model.fit(*train.Xy) # lazy: only bootstrap resampling happens here + +estim_prevalence, conf_region = model.predict_conf(test.X) +``` + +Note that `ReadMe` is computationally expensive: its cost scales with the +product of `bootstrap_trials` and `bagging_trials`, each of which requires +solving a least-squares problem. ## Composable Methods @@ -600,25 +851,192 @@ model.fit(*dataset.training.Xy) estim_prevalence = model.predict(dataset.test.X) ``` -## Confidence Regions for Class Prevalence Estimation +(confidence-regions-for-class-prevalence-estimation)= +## Quantifiers with Uncertainty Quantification -_(New in v0.2.0!)_ Some quantification methods go beyond providing a single point estimate of class prevalence values and also produce confidence regions, which characterize the uncertainty around the point estimate. In QuaPy, two such methods are currently implemented: - -* Aggregative Bootstrap: The Aggregative Bootstrap method extends any aggregative quantifier by generating confidence regions for class prevalence estimates through bootstrapping. Key features of this method include: - - * Optimized Computation: The bootstrap is applied to pre-classified instances, significantly speeding up training and inference. -During training, bootstrap repetitions are performed only after training the classifier once. These repetitions are used to train multiple aggregation functions. -During inference, bootstrap is applied over pre-classified test instances. - * General Applicability: Aggregative Bootstrap can be applied to any aggregative quantifier. - For further information, check the [example](https://github.com/HLT-ISTI/QuaPy/tree/master/examples/16.confidence_regions.py) provided. - -* BayesianCC: is a Bayesian variant of the Adjusted Classify & Count (ACC) quantifier; see more details in the [example](https://github.com/HLT-ISTI/QuaPy/tree/master/examples/14.bayesian_quantification.py) provided. +_(New in v0.2.0!)_ Some quantification methods go beyond providing a single point estimate of class prevalence values and also produce confidence regions, which characterize the uncertainty around the point estimate. In QuaPy, two such families are currently implemented: bootstrap methods and Bayesian methods. Confidence regions are constructed around a point estimate, which is typically computed as the mean value of a set of samples. -The confidence region can be instantiated in three ways: -* Confidence intervals: are standard confidence intervals generated for each class independently (_method="intervals"_). + +The confidence region can be instantiated in four ways: +* Confidence intervals: are standard confidence intervals generated for each class independently (_method="intervals"_). Since confidence intervals are independently derived for each class, Bonferroni correction can be applied. * Confidence ellipse in the simplex: an ellipse constructed around the mean point; the ellipse lies on the simplex and takes - into account possible inter-class dependencies in the data (_method="ellipse"_). + into account possible inter-class dependencies in the data (_method="ellipse"_). * Confidence ellipse in the Centered-Log Ratio (CLR) space: the underlying assumption of the ellipse is that the components are normally distributed. However, we know elements from the simplex have an inner structure. A better approach is to first - transform the components into an unconstrained space (the CLR), and then construct the ellipse in such space (_method="ellipse-clr"_). \ No newline at end of file + transform the components into an unconstrained space (the CLR), and then construct the ellipse in such space (_method="ellipse-clr"_). +* Confidence ellipse in the Isometric-Log Ratio (ILR) space: analogous to the CLR-based ellipse, but built in the ILR space + (_method="ellipse-ilr"_). + +### Aggregative Bootstrap + +The Aggregative Bootstrap method extends any aggregative quantifier by generating confidence regions for class prevalence estimates through bootstrapping. The method is described in the paper [Moreo, A., Salvati, N. + An Efficient Method for Deriving Confidence Intervals in Aggregative Quantification. + Learning to Quantify: Methods and Applications (LQ 2025), co-located at ECML-PKDD 2025. + pp 12-33, Porto (Portugal)](https://lq-2025.github.io/proceedings/CompleteVolume.pdf). + +This implementation is optimized for aggregative quantifiers. The bootstrap is applied to pre-classified instances, significantly speeding up training and inference. +During training, bootstrap repetitions are performed only after training the classifier once. These repetitions are used to train multiple aggregation functions. +During inference, bootstrap is applied over pre-classified test instances. + +Aggregative Bootstrap can be applied to any aggregative quantifier. For further information, check the [example](https://github.com/HLT-ISTI/QuaPy/tree/master/examples/16.confidence_regions.py) provided. A minimal working example is: + +```python +import quapy as qp +from quapy.method.aggregative import PACC +from quapy.method.confidence import AggregativeBootstrap + +train, test = qp.datasets.fetch_UCIMulticlassDataset('molecular').train_test + +model = AggregativeBootstrap( + PACC(), + n_test_samples=200, + confidence_level=0.95, + region='ellipse-clr', # choose among: intervals, ellipse, ellipse-clr, ellipse-ilr + random_state=0, +) +model.fit(*train.Xy) +point_estimate, conf_region = model.predict_conf(test.X) +``` + +Here `region` makes the type of uncertainty region explicit. In practice, `intervals` is often the simplest default, while `ellipse`, `ellipse-clr`, and `ellipse-ilr` provide coupled regions over the simplex. If `region='intervals'`, you can additionally set `bonferroni=True` to apply Bonferroni correction; this flag has no effect for ellipse-based regions. + +Beyond aggregative quantifiers, Bootstrap sampling can be applied to any type of quantification method, although +the speedup procedure described above is not applied. + + + + + +### Bayesian Quantification Methods + +QuaPy also provides a number of Bayesian quantifiers. While these methods are +usually related to an existing point-estimation family (e.g., ACC, EMQ/MLLS, +HDy, or KDEy), they differ enough in their goals and outputs to deserve a +separate presentation. In particular, Bayesian quantifiers typically return a +posterior mean rather than a single optimization result, expose posterior +samples or confidence regions, and often require additional probabilistic +inference dependencies. + +The optional dependencies needed for these methods can be installed with: + +```sh +pip install quapy[bayes] +``` + +#### BayesianCC (a Bayesian implementation of ACC) + +The `BayesianCC` is a variant of ACC introduced in +[Ziegler, A. and Czyż, P. "Bayesian quantification with black-box estimators", arXiv (2023)](https://arxiv.org/abs/2302.09159), +which models the probabilities `q = Mp` using latent random variables with weak Bayesian priors, rather than +plug-in probability estimates. In particular, it uses Markov Chain Monte Carlo sampling to find the values of +`p` compatible with the observed quantities. +The `aggregate` method returns the posterior mean and the `get_prevalence_samples` method can be used to find +uncertainty around `p` estimates (conditional on the observed data and the trained classifier) +and is suitable for problems in which the `q = Mp` matrix is nearly non-invertible. + +Note that this quantification method requires `val_split` to be a `float` and installation of additional dependencies (`$ pip install quapy[bayes]`) needed to run Markov chain Monte Carlo sampling. Markov Chain Monte Carlo is is slower than matrix inversion methods, but is guaranteed to sample proper probability vectors, so no clipping strategies are required. +An example presenting how to run the method and use posterior samples is available in `examples/bayesian_quantification.py`. + +#### BayesianMAPLS (a Bayesian implementation of EMQ/MLLS) + +`BayesianMAPLS` is a Bayesian variant of EMQ/MLLS proposed by +Ye, C. et al. (2024). Label shift estimation for class-imbalance problem: A +Bayesian approach. Proceedings of the IEEE/CVF Winter Conference on +Applications of Computer Vision (WACV 2024). QuaPy's implementation is +adapted from the [authors' reference code](https://github.com/ChangkunYe/MAPLS/blob/main/label_shift/mapls.py). +Rather than returning a single point estimate for the class prevalence, it +places a Dirichlet prior over the sought prevalence vector (in an +unconstrained, Isometric-Log-Ratio-transformed space) and samples from the +resulting posterior via Markov Chain Monte Carlo (using `numpyro`/`jax`), +conditioned on a preliminary MAP estimate obtained via the underlying `mapls` +routine. Like `BayesianCC`, its `aggregate` method returns the posterior mean, +while `predict_conf` additionally returns a confidence region (`intervals`, +`ellipse`, `ellipse-clr`, or `ellipse-ilr`) built from the posterior samples. +For interval regions, all Bayesian methods also accept `bonferroni=True` to +apply Bonferroni correction; this flag has no effect for ellipse-based regions. + +This method requires installation of additional dependencies +(`$ pip install quapy[bayes]`) needed to run MCMC sampling; parameters +`num_warmup` and `num_samples` control the length of the chain, and `prior` +allows choosing between a uniform Dirichlet prior (default) or one of the +data-dependent priors ("map"/"map2") proposed in the original paper. + +```python +import quapy as qp +from quapy.method._bayesian import BayesianMAPLS +from sklearn.linear_model import LogisticRegression + +train, test = qp.datasets.fetch_UCIBinaryDataset('haberman').train_test + +model = BayesianMAPLS(LogisticRegression()) +model.fit(*train.Xy) +estim_prevalence, conf_region = model.predict_conf(test.X) +``` + +#### PQ: Precise Quantifier (a Bayesian implementation of HDy) + +`PQ` (Precise Quantifier), available at `qp.method.confidence.PQ`, is a +Bayesian distribution-matching variant of `HDy` proposed in +[Igiraneza, A.B., Fraser, C., and Hinch, R. (2025). Estimating prevalence +with precision and accuracy.](https://arxiv.org/abs/2507.06061) +Rather than matching a single test histogram against a mixture of two +class-conditional histograms via a divergence measure (as `HDy` does), `PQ` +places the histogram-matching problem in a Bayesian setting and samples the +full posterior distribution over the (binary) prevalence value via Markov +Chain Monte Carlo (using `stan`). Its `aggregate` method returns the +posterior mean, while `predict_conf` additionally returns a confidence +region built from the posterior samples (`intervals`, `ellipse`, or +`ellipse-clr`). + +`PQ` accepts `nbins` (the number of histogram bins, quantile-based by default, +or uniform if `fixed_bins=True`), and the usual MCMC controls `num_warmup`, +`num_samples`, and `stan_seed`. This method relies on the optional `stan` +dependency, installed via `$ pip install quapy[bayes]`. + +```python +import quapy as qp +from quapy.method.confidence import PQ +from sklearn.linear_model import LogisticRegression + +train, test = qp.datasets.fetch_UCIBinaryDataset('haberman').train_test + +model = PQ(LogisticRegression()) +model.fit(*train.Xy) +estim_prevalence, conf_region = model.predict_conf(test.X) +``` + +#### BayesianKDEy (a Bayesian implementation of KDEyML) + +`BayesianKDEy`, available at `qp.method._bayesian.BayesianKDEy`, is a Bayesian +version of KDEy proposed by [Moreo et al. 2026](https://arxiv.org/abs/2607.04977). +Instead of solving for the single prevalence vector that +minimizes a divergence between the test distribution and a KDE-based mixture +model (as the KDEy variants above do), `BayesianKDEy` places a Dirichlet +prior over the prevalence vector and samples its posterior via Markov Chain +Monte Carlo (using `numpyro`/`jax`), conditioned on the same KDE mixture +components. Its `aggregate` method returns the posterior mean, while +`predict_conf` additionally returns a confidence region built from the +posterior samples. + +In addition to the `kernel` and `bandwidth` hyperparameters (with the same +`gaussian`/`aitchison`/`ilr` kernel choice, and `shrinkage` regularization +for the latter two, available in `KDEyML`), `BayesianKDEy` exposes the usual +MCMC controls: `num_warmup`, `num_samples`, `mcmc_seed`, a `temperature` for +posterior calibration, and `prior` for choosing the Dirichlet prior +(`'uniform'` by default, or a custom scalar/array). This method relies on +the optional MCMC dependencies, installed via +`$ pip install quapy[bayes]`. + +```python +import quapy as qp +from quapy.method._bayesian import BayesianKDEy +from sklearn.linear_model import LogisticRegression + +train, test = qp.datasets.fetch_UCIBinaryDataset('haberman').train_test + +model = BayesianKDEy(LogisticRegression(), bandwidth=0.1) +model.fit(*train.Xy) +estim_prevalence, conf_region = model.predict_conf(test.X) +``` + diff --git a/docs/source/manuals/plots/app-grid.png b/docs/source/manuals/plots/app-grid.png new file mode 100644 index 0000000..d8cce7a Binary files /dev/null and b/docs/source/manuals/plots/app-grid.png differ diff --git a/docs/source/manuals/plots/app-kraemer.png b/docs/source/manuals/plots/app-kraemer.png new file mode 100644 index 0000000..1c842d1 Binary files /dev/null and b/docs/source/manuals/plots/app-kraemer.png differ diff --git a/docs/source/manuals/plots/dirichlet.png b/docs/source/manuals/plots/dirichlet.png new file mode 100644 index 0000000..664ffc6 Binary files /dev/null and b/docs/source/manuals/plots/dirichlet.png differ diff --git a/docs/source/manuals/plots/npp.png b/docs/source/manuals/plots/npp.png new file mode 100644 index 0000000..b43d382 Binary files /dev/null and b/docs/source/manuals/plots/npp.png differ diff --git a/docs/source/manuals/plots/simplex_visualization.png b/docs/source/manuals/plots/simplex_visualization.png new file mode 100644 index 0000000..2f6c570 Binary files /dev/null and b/docs/source/manuals/plots/simplex_visualization.png differ diff --git a/docs/source/manuals/plotting.md b/docs/source/manuals/plotting.md index 67f9f16..153ccb4 100644 --- a/docs/source/manuals/plotting.md +++ b/docs/source/manuals/plotting.md @@ -251,3 +251,41 @@ In those cases, however, it is likely that the variances of each method get higher, to the detriment of the visualization. We recommend to set _show_std=False_ in those cases in order to hide the color bands. + +## Simplex Visualisation + +For three-class problems, prevalence vectors lie on the 2-dimensional probability simplex. +The function `qp.plot.plot_simplex` provides a lightweight ternary plot that can combine +scatter layers, shaded regions, and density overlays. + +A simplex plot is specified through optional layers. Point layers are dictionaries with +fields `points`, `label`, and `style`; region layers are dictionaries with fields `fn`, +`label`, `color`, and `alpha`. + +```python +import numpy as np +import quapy as qp + +true_prev = np.array([0.20, 0.35, 0.45]) +train_prev = np.array([0.50, 0.30, 0.20]) +posterior_cloud = np.random.default_rng(0).dirichlet(alpha=40 * true_prev, size=200) + +qp.plot.plot_simplex( + point_layers=[ + {'points': posterior_cloud, 'label': 'posterior cloud', 'style': {'s': 12, 'alpha': 0.25, 'color': 'steelblue'}}, + {'points': true_prev, 'label': 'true prevalence', 'style': {'s': 90, 'color': 'black'}}, + {'points': train_prev, 'label': 'training prevalence', 'style': {'s': 90, 'color': 'darkorange'}}, + ], + density_function=lambda p: np.exp(-40 * np.sum((p - true_prev) ** 2, axis=1)), + class_names=['class A', 'class B', 'class C'], + savepath='./plots/simplex.png', +) +``` + +See the dedicated +[example](https://github.com/HLT-ISTI/QuaPy/blob/master/examples/19.visualizing_simplex.py) +for a slightly richer illustration. The current example combines a posterior +cloud, the true/training/predicted prevalences, a smooth density surface, and +a region induced by Bonferroni-corrected 95% confidence intervals. + +![simplex visualization](./plots/simplex_visualization.png) diff --git a/docs/source/manuals/protocols.md b/docs/source/manuals/protocols.md index 17bc41a..65fa62d 100644 --- a/docs/source/manuals/protocols.md +++ b/docs/source/manuals/protocols.md @@ -33,7 +33,7 @@ QuaPy provides implementations of most popular sample generation protocols used in literature. This is the subject of the following sections. -## Artificial-Prevalence Protocol +## APP: Artificial-Prevalence Protocol The "artificial-sampling protocol" (APP) proposed by [Forman (2005)](https://link.springer.com/chapter/10.1007/11564096_55) @@ -43,21 +43,21 @@ desired prevalence values covering the full spectrum. In APP, the user specifies the number of (equally distant) points to be generated from the interval [0,1]; -in QuaPy this is achieved by setting _n_prevpoints_. -For example, if _n_prevpoints=11_ then, for each class, the prevalence values +in QuaPy this is achieved by setting _n_prevalences_. +For example, if _n_prevalences=11_ then, for each class, the prevalence values [0., 0.1, 0.2, ..., 1.] will be used. This means that, for two classes, the number of different prevalence values will be 11 (since, once the prevalence of one class is determined, the other one is constrained). For 3 classes, the number of valid combinations can be obtained as 11 + 10 + ... + 1 = 66. In general, the number of valid combinations that will be produced for a given -value of n_prevpoints can be consulted by invoking +value of _n_prevalences_ can be consulted by invoking _num_prevalence_combinations_, e.g.: ```python import quapy.functional as F -n_prevpoints = 21 +n_prevalences = 21 n_classes = 4 -n = F.num_prevalence_combinations(n_prevpoints, n_classes, n_repeats=1) +n = F.num_prevalence_combinations(n_prevalences, n_classes, n_repeats=1) ``` in this example, _n=1771_. Note the last argument, _n_repeats_, that @@ -74,13 +74,13 @@ _get_nprevpoints_approximation_, e.g.: ```python budget = 5000 -n_prevpoints = F.get_nprevpoints_approximation(budget, n_classes, n_repeats=1) -n = F.num_prevalence_combinations(n_prevpoints, n_classes, n_repeats=1) -print(f'by setting n_prevpoints={n_prevpoints} the number of evaluations for {n_classes} classes will be {n}') +n_prevalences = F.get_nprevpoints_approximation(budget, n_classes, n_repeats=1) +n = F.num_prevalence_combinations(n_prevalences, n_classes, n_repeats=1) +print(f'by setting n_prevalences={n_prevalences} the number of evaluations for {n_classes} classes will be {n}') ``` this will produce the following output: ``` -by setting n_prevpoints=30 the number of evaluations for 4 classes will be 4960 +by setting n_prevalences=30 the number of evaluations for 4 classes will be 4960 ``` The following code shows an example of usage of APP for model selection @@ -129,7 +129,16 @@ in such cases QuaPy takes the value of _qp.environ['SAMPLE_SIZE']_. This protocol is useful for testing a quantifier under conditions of _prior probability shift_. -## Sampling from the unit-simplex, the Uniform-Prevalence Protocol (UPP) +The following ternary plot, generated by [example 21](https://github.com/HLT-ISTI/QuaPy/blob/master/examples/21.visualizing_protocols.py), +shows the prevalence values covered by a grid-based APP in a three-class problem (`academic-success`): + +![APP grid protocol](./plots/app-grid.png) + +Each point corresponds to one sampled prevalence vector. As expected, the +points lie on a regular grid over the simplex, ensuring systematic coverage of +the prevalence space. + +## UPP: Sampling from the unit-simplex, the Uniform-Prevalence Protocol Generating all possible combinations from a grid of prevalence values (APP) in multiclass is cumbersome, and when the number of classes increases it rapidly @@ -148,7 +157,7 @@ for sampling from the unit-simplex as many vectors of prevalence values as indic in the _repeats_ parameter. UPP can be instantiated as: ```python -protocol = qp.in_protocol.UPP(test, repeats=100) +protocol = qp.protocol.UPP(test, repeats=100) ``` This is the most convenient protocol for datasets @@ -157,8 +166,22 @@ containing many classes; see, e.g., and is useful for testing a quantifier under conditions of _prior probability shift_. +The next plot shows one such protocol, labelled in example 21 as +_APP(Kraemer)_, to emphasize that it plays the role of an artificial-prevalence +protocol without relying on a fixed grid: -## Natural-Prevalence Protocol +![UPP / APP Kraemer protocol](./plots/app-kraemer.png) + +Unlike grid-based APP, UPP does not force prevalence vectors to lie on a +regular lattice. Instead, it spreads samples over the simplex in a +statistically uniform way, making it attractive when the number of classes is +large and exhaustive grids become impractical. + +*Note* that UPP is actually a different (modern) implementation of the Artificial Prevalence Protocol, +and is here given a different name simply to allow both implementations coexist in QuaPy. "UPP" is not a +proper accademic name, and practitioners should rather refer to it as APP. + +## NPP: Natural-Prevalence Protocol The "natural-prevalence protocol" (NPP) comes down to generating samples drawn uniformly at random from the original labelled collection. This protocol has @@ -168,9 +191,47 @@ All other things being equal, this protocol can be used just like APP or UPP, and is instantiated via: ```python -protocol = qp.in_protocol.NPP(test, repeats=100) +protocol = qp.protocol.NPP(test, repeats=100) ``` +The prevalence coverage of NPP is much more concentrated, since the samples are +obtained by plain random subsampling from the test set and therefore remain +close to its natural prevalence: + +![Natural-prevalence protocol](./plots/npp.png) + +This makes NPP useful when one wants to evaluate quantifiers under mild or +realistic drift conditions, but much less suitable than APP or UPP for stress +testing performance across the full simplex. + +## Dirichlet Protocol + +QuaPy also implements a :class:`DirichletProtocol`, which samples prevalence +vectors from a Dirichlet distribution before drawing the corresponding sample +from the labelled collection: + +```python +protocol = qp.protocol.DirichletProtocol(test, alpha=0.2, repeats=100) +``` + +The parameter `alpha` controls how concentrated the protocol is. Small values +of `alpha` favour sparse prevalence vectors near the corners of the simplex, +while larger values generate more balanced mixtures. When all entries of +`alpha` are equal to 1, the protocol becomes uniformly distributed over the +simplex, similarly in spirit to UPP. + +The following plot shows the effect of a sparse prior with `alpha=0.2`: + +![Dirichlet protocol](./plots/dirichlet.png) + +Compared to UPP, the mass is clearly pulled towards the vertices and edges, +thus producing more extreme label-shift scenarios. + +The parameter `alpha` in the Dirichlet distribution is typically defined as an +array of shape `(n_classes)`. When the user specifies a single value, QuaPy +broadcasts this value for all classes. Conversely, a different value can be +specified for each class. + ## Other protocols Other protocols exist in QuaPy and will be added to the `qp.protocol.py` module. \ No newline at end of file diff --git a/docs/source/quapy.method.rst b/docs/source/quapy.method.rst index ac0dfc8..88fcc7d 100644 --- a/docs/source/quapy.method.rst +++ b/docs/source/quapy.method.rst @@ -60,6 +60,14 @@ quapy.method.composable module :undoc-members: :show-inheritance: +quapy.method.confidence module +------------------------------ + +.. automodule:: quapy.method.confidence + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/examples/16.confidence_regions.py b/examples/16.confidence_regions.py index 27fbfbd..4fb5c72 100644 --- a/examples/16.confidence_regions.py +++ b/examples/16.confidence_regions.py @@ -36,7 +36,7 @@ with qp.util.temp_seed(0): true_prev = shifted_test.prevalence() # by calling "quantify_conf", we obtain the point estimate and the confidence intervals around it - pred_prev, conf_intervals = pacc.quantify_conf(shifted_test.X) + pred_prev, conf_intervals = pacc.predict_conf(shifted_test.X) # conf_intervals is an instance of ConfidenceRegionABC, which provides some useful utilities like: # - coverage: a function which computes the fraction of true values that belong to the confidence region @@ -75,7 +75,8 @@ There are different ways for constructing confidence regions implemented in QuaP convenient for taking into account the inner structure of the probability simplex) use: AggregativeBootstrap(PACC(), confidence_level=0.95, method='ellipse-clr') -Other methods that return confidence regions in QuaPy include the BayesianCC method. +Other methods that return confidence regions in QuaPy include the Bayesian methods (BayesianCC, + BayesianMAPLS, PQ, and BayesianKDEy). """ diff --git a/examples/18.ReadMe_for_text_analysis.py b/examples/18.ReadMe_for_text_analysis.py new file mode 100644 index 0000000..7a70022 --- /dev/null +++ b/examples/18.ReadMe_for_text_analysis.py @@ -0,0 +1,60 @@ +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.feature_selection import SelectKBest, chi2 + +import quapy as qp +from quapy.method.non_aggregative import ReadMe +import quapy.functional as F +from sklearn.pipeline import Pipeline + +""" +This example showcases how to use the non-aggregative method ReadMe proposed by Hopkins and King. +This method is for text analysis, so let us first instantiate a dataset for sentiment quantification (we +use IMDb for this example). The method is quite computationally expensive, so we will restrict the training +set to 1000 documents only. +""" +reviews = qp.datasets.fetch_reviews('imdb').reduce(n_train=1000, random_state=0) + +""" +We need to convert text to bag-of-words representations. Actually, ReadMe requires the representations to be +binary (i.e., storing a 1 whenever a document contains certain word, or 0 otherwise), so we will not use +TFIDF weighting. We will also retain the top 1000 most important features according to chi2. +""" +encode_0_1 = Pipeline([ + ('0_1_terms', CountVectorizer(min_df=5, binary=True)), + ('feat_sel', SelectKBest(chi2, k=1000)) +]) +train, test = qp.data.preprocessing.instance_transformation(reviews, encode_0_1, inplace=True).train_test + +""" +We now instantiate ReadMe, with the prob_model='full' (default behaviour, implementing the Hopkins and King original +idea). This method consists of estimating Q(Y) by solving: + +Q(X) = \sum_i Q(X|Y=i) Q(Y=i) + +without resorting to estimating the posteriors Q(Y=i|X), by solving a linear least-squares problem. +However, since Q(X) and Q(X|Y=i) are matrices of shape (2^K, 1) and (2^K, n), with K the number of features +and n the number of classes, their calculation becomes intractable. ReadMe instead performs bagging (i.e., it +samples small sets of features and averages the results) thus reducing K to a few terms. In our example we +set K (bagging_range) to 20, and the number of bagging_trials to 100. + +ReadMe also computes confidence intervals via bootstrap. We set the number of bootstrap trials to 100. +""" +readme = ReadMe(prob_model='full', bootstrap_trials=100, bagging_trials=100, bagging_range=20, random_state=0, verbose=True) +readme.fit(*train.Xy) # <- there is actually nothing happening here (only bootstrap resampling); the method is "lazy" + # and postpones most of the calculations to the test phase. + +# since the method is slow, we will only test 3 cases with different imbalances +few_negatives = [0.25, 0.75] +balanced = [0.5, 0.5] +few_positives = [0.75, 0.25] + +for test_prev in [few_negatives, balanced, few_positives]: + sample = reviews.test.sampling(500, *test_prev, random_state=0) # draw sets of 500 documents with desired prevs + prev_estim, conf = readme.predict_conf(sample.X) + err = qp.error.mae(sample.prevalence(), prev_estim) + print(f'true-prevalence={F.strprev(sample.prevalence())},\n' + f'predicted-prevalence={F.strprev(prev_estim)}, with confidence intervals {conf},\n' + f'MAE={err:.4f}') + + + diff --git a/examples/19.visualizing_simplex.py b/examples/19.visualizing_simplex.py new file mode 100644 index 0000000..1ec5f9c --- /dev/null +++ b/examples/19.visualizing_simplex.py @@ -0,0 +1,68 @@ +import numpy as np +import quapy as qp +from quapy.method.confidence import ConfidenceIntervals + + +""" +A minimal example showing how to visualise ternary prevalences on the simplex. +The plot combines a cloud of posterior triplets, a few reference prevalences, a +confidence ellipse induced by the cloud, and a smooth density centred around the +true prevalence. +""" + +rng = np.random.default_rng(0) +true_prev = np.array([0.20, 0.35, 0.45]) +train_prev = np.array([0.50, 0.30, 0.20]) +pred_prev = np.array([0.18, 0.39, 0.43]) +posterior_cloud = rng.dirichlet(alpha=45 * true_prev, size=250) + +point_layers = [ + { + 'points': posterior_cloud, + 'label': 'posterior cloud', + 'style': {'s': 12, 'alpha': 0.25, 'color': 'steelblue', 'edgecolors': 'none'}, + }, + { + 'points': true_prev, + 'label': 'true prevalence', + 'style': {'s': 70, 'color': 'black'}, + }, + { + 'points': pred_prev, + 'label': 'predicted prevalence', + 'style': {'s': 70, 'color': 'crimson'}, + }, + { + 'points': train_prev, + 'label': 'training prevalence', + 'style': {'s': 70, 'color': 'darkorange'}, + }, +] + +confidence_region = ConfidenceIntervals(posterior_cloud, confidence_level=0.95, bonferroni_correction=True) + +region_layers = [ + { + 'fn': lambda p: float(p in confidence_region), + 'label': '95% confidence intervals', + 'color': 'seagreen', + 'alpha': 0.15, + } +] + +density = lambda p: np.exp(-45 * np.sum((p - true_prev) ** 2, axis=1)) + +qp.plot.plot_simplex( + point_layers=point_layers, + region_layers=region_layers, + density_function=density, + density_color='royalblue', + class_names=['class A', 'class B', 'class C'], + title='Ternary prevalence visualisation', + legend_ncol=3, + figsize=(7.2, 5.8), + class_name_fontsize=9, + title_fontsize=10, + legend_fontsize=8, + savepath='./plots/simplex_visualization.png', +) diff --git a/examples/20.cifar10_quantification.py b/examples/20.cifar10_quantification.py new file mode 100644 index 0000000..55e8f3f --- /dev/null +++ b/examples/20.cifar10_quantification.py @@ -0,0 +1,64 @@ +import quapy as qp +from quapy.data.datasets import fetch_image_embeddings +from quapy.method.aggregative import EMQ, RLLS +from quapy.classification.calibration import TemperatureScalingFromLogits +from quapy.protocol import UPP +from sklearn.linear_model import LogisticRegression + + +# This example illustrates how to run experiments with image datasets, in this case with CIFAR10 +# The datasets available in quapy do not consist of raw image files, but are instead pre-generated +# embeddings (see the manuals for further information). + +if __name__ == '__main__': + + # Let us begin with a typical case in which the embeddings come from the penultimate layer of a neural + # model (in this case, a resnet18). We get these representations by specifying embedding='features' + + print('fetching cifar10 embeddings') + train, test = fetch_image_embeddings(dataset_name='cifar10', embedding='features').train_test + + print('training:', train) + print('test:', test) + + Xtr, ytr = train.Xy + + # let us train an Expectation Maximazion Quantifier (EMQ), aka Maximum Likelihood for Label Shift (MLLS) + # using a logistic regressor as the underlying classifier, with Bias Corrected Temperature Scaling (BCTS) + + bcts_emq = EMQ(classifier=LogisticRegression(), calib='bcts', val_split=5) + + print(f'fitting quantifier {bcts_emq}') + bcts_emq.fit(Xtr, ytr) + + # we generate many samples exhibiting prior probability shift with the artificial prevalence protocol + # (we use the multiclass variant UPP instead of the grid-based APP) + + qp.environ["SAMPLE_SIZE"] = 500 # when the sample size is common to all experiments, it is conveniet to set it once and for all + artificial_prev_prot = UPP(test, repeats=200) + print('generating 200 test bags of 500 instances each') + + bctsemq_report = qp.evaluation.evaluation_report(bcts_emq, protocol=artificial_prev_prot, error_metrics=['mae', 'mrae']) + print(bctsemq_report.mean(numeric_only=True)) + + # we could instead use the pre-generated logits of the resnet18 + print('fetching cifar10 logits') + train, test = fetch_image_embeddings(dataset_name='cifar10', embedding='logits').train_test + Xtr, ytr = train.Xy + + # in this case, the representations are already classification-related outputs; + # we can convert them into (hopefully well-) calibrated outputs via TemperatureScaling + + print('generating posterior probabilities out of logits via temperature scaling') + emq = EMQ(classifier=TemperatureScalingFromLogits(bias_corrected=True)) + emq.fit(Xtr, ytr) + + print('generating 200 test bags of 500 instances each') + artificial_prev_prot = UPP(test, repeats=200) + emq_report = qp.evaluation.evaluation_report(emq, protocol=artificial_prev_prot, error_metrics=['mae', 'mrae']) + print(emq_report.mean(numeric_only=True)) + + + + + diff --git a/examples/21.visualizing_protocols.py b/examples/21.visualizing_protocols.py new file mode 100644 index 0000000..8f87e69 --- /dev/null +++ b/examples/21.visualizing_protocols.py @@ -0,0 +1,51 @@ +import numpy as np +import quapy as qp +from quapy.data.datasets import fetch_UCIMulticlassDataset +from quapy.protocol import APP, NPP, UPP, DirichletProtocol + + +""" +Ternary plots showcasing different sampling protocols. +""" + +rng = np.random.default_rng(0) + +train, test = fetch_UCIMulticlassDataset(dataset_name='academic-success').train_test + +train_prev = { + 'points': train.prevalence(), + 'label': 'training prevalence', + 'style': {'s': 70, 'color': 'darkorange'}, +} + +def protocols(): + yield 'app-grid', 'Artificial Prevalence Protocol (grid)', APP(test, n_prevalences=21, repeats=1, sample_size=100) + yield 'app-kraemer', 'Artificial Prevalence Protocol (Kraemer)', UPP(test, repeats=5000, sample_size=500) + yield 'npp', 'Natural Prevalence Protocol', NPP(test, repeats=1000, sample_size=100) + yield 'dirichlet', 'Dirichlet(alpha=0.2)', DirichletProtocol(test, alpha=0.2, repeats=5000, sample_size=100) + +for file_name, prot_name, protocol in protocols(): + app_points = { + 'points': [prev for _, prev in protocol()], + 'label': prot_name, + 'style': {'s': 15, 'alpha': 0.5, 'color': 'steelblue', 'edgecolors': 'none'}, + } + + point_layers = [ + app_points, + train_prev, + ] + + dispersion = 0.1 + + qp.plot.plot_simplex( + point_layers=point_layers, + class_names=['class A', 'class B', 'class C'], + #title='Ternary prevalence visualisation', + legend_ncol=3, + figsize=(7.2, 5.8), + class_name_fontsize=9, + title_fontsize=10, + legend_fontsize=8, + savepath=f'./plots/{file_name}.png', + ) diff --git a/experimental_non_aggregative/custom_vectorizers.py b/experimental_non_aggregative/custom_vectorizers.py new file mode 100644 index 0000000..13337b9 --- /dev/null +++ b/experimental_non_aggregative/custom_vectorizers.py @@ -0,0 +1,254 @@ +from scipy.sparse import csc_matrix, csr_matrix +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer, CountVectorizer +import numpy as np +from joblib import Parallel, delayed +import sklearn +import math +from scipy.stats import t + + +class ContTable: + def __init__(self, tp=0, tn=0, fp=0, fn=0): + self.tp=tp + self.tn=tn + self.fp=fp + self.fn=fn + + def get_d(self): return self.tp + self.tn + self.fp + self.fn + + def get_c(self): return self.tp + self.fn + + def get_not_c(self): return self.tn + self.fp + + def get_f(self): return self.tp + self.fp + + def get_not_f(self): return self.tn + self.fn + + def p_c(self): return (1.0*self.get_c())/self.get_d() + + def p_not_c(self): return 1.0-self.p_c() + + def p_f(self): return (1.0*self.get_f())/self.get_d() + + def p_not_f(self): return 1.0-self.p_f() + + def p_tp(self): return (1.0*self.tp) / self.get_d() + + def p_tn(self): return (1.0*self.tn) / self.get_d() + + def p_fp(self): return (1.0*self.fp) / self.get_d() + + def p_fn(self): return (1.0*self.fn) / self.get_d() + + def tpr(self): + c = 1.0*self.get_c() + return self.tp / c if c > 0.0 else 0.0 + + def fpr(self): + _c = 1.0*self.get_not_c() + return self.fp / _c if _c > 0.0 else 0.0 + + +def __ig_factor(p_tc, p_t, p_c): + den = p_t * p_c + if den != 0.0 and p_tc != 0: + return p_tc * math.log(p_tc / den, 2) + else: + return 0.0 + + +def information_gain(cell): + return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + \ + __ig_factor(cell.p_fp(), cell.p_f(), cell.p_not_c()) +\ + __ig_factor(cell.p_fn(), cell.p_not_f(), cell.p_c()) + \ + __ig_factor(cell.p_tn(), cell.p_not_f(), cell.p_not_c()) + + +def squared_information_gain(cell): + return information_gain(cell)**2 + + +def posneg_information_gain(cell): + ig = information_gain(cell) + if cell.tpr() < cell.fpr(): + return -ig + else: + return ig + + +def pos_information_gain(cell): + if cell.tpr() < cell.fpr(): + return 0 + else: + return information_gain(cell) + +def pointwise_mutual_information(cell): + return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + + +def gss(cell): + return cell.p_tp()*cell.p_tn() - cell.p_fp()*cell.p_fn() + + +def chi_square(cell): + den = cell.p_f() * cell.p_not_f() * cell.p_c() * cell.p_not_c() + if den==0.0: return 0.0 + num = gss(cell)**2 + return num / den + + +def conf_interval(xt, n): + if n>30: + z2 = 3.84145882069 # norm.ppf(0.5+0.95/2.0)**2 + else: + z2 = t.ppf(0.5 + 0.95 / 2.0, df=max(n-1,1)) ** 2 + p = (xt + 0.5 * z2) / (n + z2) + amplitude = 0.5 * z2 * math.sqrt((p * (1.0 - p)) / (n + z2)) + return p, amplitude + + +def strength(minPosRelFreq, minPos, maxNeg): + if minPos > maxNeg: + return math.log(2.0 * minPosRelFreq, 2.0) + else: + return 0.0 + + +#set cancel_features=True to allow some features to be weighted as 0 (as in the original article) +#however, for some extremely imbalanced dataset caused all documents to be 0 +def conf_weight(cell, cancel_features=False): + c = cell.get_c() + not_c = cell.get_not_c() + tp = cell.tp + fp = cell.fp + + pos_p, pos_amp = conf_interval(tp, c) + neg_p, neg_amp = conf_interval(fp, not_c) + + min_pos = pos_p-pos_amp + max_neg = neg_p+neg_amp + den = (min_pos + max_neg) + minpos_relfreq = min_pos / (den if den != 0 else 1) + + str_tplus = strength(minpos_relfreq, min_pos, max_neg); + + if str_tplus == 0 and not cancel_features: + return 1e-20 + + return str_tplus + + +def get_tsr_matrix(cell_matrix, tsr_score_funtion): + nC = len(cell_matrix) + nF = len(cell_matrix[0]) + tsr_matrix = [[tsr_score_funtion(cell_matrix[c,f]) for f in range(nF)] for c in range(nC)] + return np.array(tsr_matrix) + + +def feature_label_contingency_table(positive_document_indexes, feature_document_indexes, nD): + tp_ = len(positive_document_indexes & feature_document_indexes) + fp_ = len(feature_document_indexes - positive_document_indexes) + fn_ = len(positive_document_indexes - feature_document_indexes) + tn_ = nD - (tp_ + fp_ + fn_) + return ContTable(tp=tp_, tn=tn_, fp=fp_, fn=fn_) + + +def category_tables(feature_sets, category_sets, c, nD, nF): + return [feature_label_contingency_table(category_sets[c], feature_sets[f], nD) for f in range(nF)] + + +def get_supervised_matrix(coocurrence_matrix, label_matrix, n_jobs=-1): + """ + Computes the nC x nF supervised matrix M where Mcf is the 4-cell contingency table for feature f and class c. + Efficiency O(nF x nC x log(S)) where S is the sparse factor + """ + + nD, nF = coocurrence_matrix.shape + nD2, nC = label_matrix.shape + + if nD != nD2: + raise ValueError('Number of rows in coocurrence matrix shape %s and label matrix shape %s is not consistent' % + (coocurrence_matrix.shape,label_matrix.shape)) + + def nonzero_set(matrix, col): + return set(matrix[:, col].nonzero()[0]) + + if isinstance(coocurrence_matrix, csr_matrix): + coocurrence_matrix = csc_matrix(coocurrence_matrix) + feature_sets = [nonzero_set(coocurrence_matrix, f) for f in range(nF)] + category_sets = [nonzero_set(label_matrix, c) for c in range(nC)] + cell_matrix = Parallel(n_jobs=n_jobs, backend="threading")( + delayed(category_tables)(feature_sets, category_sets, c, nD, nF) for c in range(nC) + ) + return np.array(cell_matrix) + + +class TSRweighting(BaseEstimator,TransformerMixin): + """ + Supervised Term Weighting function based on any Term Selection Reduction (TSR) function (e.g., information gain, + chi-square, etc.) or, more generally, on any function that could be computed on the 4-cell contingency table for + each category-feature pair. + The supervised_4cell_matrix is a `(n_classes, n_words)` matrix containing the 4-cell contingency tables + for each class-word pair, and can be pre-computed (e.g., during the feature selection phase) and passed as an + argument. + When `n_classes>1`, i.e., in multiclass scenarios, a global_policy is used in order to determine a + single feature-score which informs about its relevance. Accepted policies include "max" (takes the max score + across categories), "ave" and "wave" (take the average, or weighted average, across all categories -- weights + correspond to the class prevalence), and "sum" (which sums all category scores). + """ + + def __init__(self, tsr_function, global_policy='max', supervised_4cell_matrix=None, sublinear_tf=True, norm='l2', min_df=3, n_jobs=-1): + if global_policy not in ['max', 'ave', 'wave', 'sum']: raise ValueError('Global policy should be in {"max", "ave", "wave", "sum"}') + self.tsr_function = tsr_function + self.global_policy = global_policy + self.supervised_4cell_matrix = supervised_4cell_matrix + self.sublinear_tf = sublinear_tf + self.norm = norm + self.min_df = min_df + self.n_jobs = n_jobs + + def fit(self, X, y): + self.count_vectorizer = CountVectorizer(min_df=self.min_df) + X = self.count_vectorizer.fit_transform(X) + + self.tf_vectorizer = TfidfTransformer( + norm=None, use_idf=False, smooth_idf=False, sublinear_tf=self.sublinear_tf + ).fit(X) + + if len(y.shape) == 1: + y = np.expand_dims(y, axis=1) + + nD, nC = y.shape + nF = len(self.tf_vectorizer.get_feature_names_out()) + + if self.supervised_4cell_matrix is None: + self.supervised_4cell_matrix = get_supervised_matrix(X, y, n_jobs=self.n_jobs) + else: + if self.supervised_4cell_matrix.shape != (nC, nF): + raise ValueError("Shape of supervised information matrix is inconsistent with X and y") + + tsr_matrix = get_tsr_matrix(self.supervised_4cell_matrix, self.tsr_function) + + if self.global_policy == 'ave': + self.global_tsr_vector = np.average(tsr_matrix, axis=0) + elif self.global_policy == 'wave': + category_prevalences = [sum(y[:,c])*1.0/nD for c in range(nC)] + self.global_tsr_vector = np.average(tsr_matrix, axis=0, weights=category_prevalences) + elif self.global_policy == 'sum': + self.global_tsr_vector = np.sum(tsr_matrix, axis=0) + elif self.global_policy == 'max': + self.global_tsr_vector = np.amax(tsr_matrix, axis=0) + return self + + def fit_transform(self, X, y): + return self.fit(X,y).transform(X) + + def transform(self, X): + if not hasattr(self, 'global_tsr_vector'): raise NameError('TSRweighting: transform method called before fit.') + X = self.count_vectorizer.transform(X) + tf_X = self.tf_vectorizer.transform(X).toarray() + weighted_X = np.multiply(tf_X, self.global_tsr_vector) + if self.norm is not None and self.norm!='none': + weighted_X = sklearn.preprocessing.normalize(weighted_X, norm=self.norm, axis=1, copy=False) + return csr_matrix(weighted_X) diff --git a/experimental_non_aggregative/method_dxs.py b/experimental_non_aggregative/method_dxs.py new file mode 100644 index 0000000..93fb67e --- /dev/null +++ b/experimental_non_aggregative/method_dxs.py @@ -0,0 +1,208 @@ +from scipy.sparse import issparse +from sklearn.decomposition import TruncatedSVD +from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.preprocessing import StandardScaler + +import quapy as qp +from data import LabelledCollection +import numpy as np + +from experimental_non_aggregative.custom_vectorizers import * +from method._kdey import KDEBase +from protocol import APP +from quapy.method.aggregative import HDy, DistributionMatchingY +from quapy.method.base import BaseQuantifier +from scipy import optimize +import pandas as pd +import quapy.functional as F + + +# TODO: explore the bernoulli (term presence/absence) variant +# TODO: explore the multinomial (term frequency) variant +# TODO: explore the multinomial + length normalization variant +# TODO: consolidate the TSR-variant (e.g., using information gain) variant; +# - works better with the idf? +# - works better with length normalization? +# - etc + +class DxS(BaseQuantifier): + def __init__(self, vectorizer=None, divergence='topsoe'): + self.vectorizer = vectorizer + self.divergence = divergence + + # def __as_distribution(self, instances): + # return np.asarray(instances.sum(axis=0) / instances.sum()).flatten() + + def __as_distribution(self, instances): + dist = instances.mean(axis=0) + return np.asarray(dist).flatten() + + def fit(self, text_instances, labels): + + classes = np.unique(labels) + + if self.vectorizer is not None: + text_instances = self.vectorizer.fit_transform(text_instances, y=labels) + + distributions = [] + for class_i in classes: + distributions.append(self.__as_distribution(text_instances[labels == class_i])) + + self.validation_distribution = np.asarray(distributions) + + return self + + def predict(self, text_instances): + if self.vectorizer is not None: + text_instances = self.vectorizer.transform(text_instances) + + test_distribution = self.__as_distribution(text_instances) + divergence = qp.functional.get_divergence(self.divergence) + n_classes, n_feats = self.validation_distribution.shape + + def match(prev): + prev = np.expand_dims(prev, axis=0) + mixture_distribution = (prev @ self.validation_distribution).flatten() + return divergence(test_distribution, mixture_distribution) + + # the initial point is set as the uniform distribution + uniform_distribution = np.full(fill_value=1 / n_classes, shape=(n_classes,)) + + # solutions are bounded to those contained in the unit-simplex + bounds = tuple((0, 1) for x in range(n_classes)) # values in [0,1] + constraints = ({'type': 'eq', 'fun': lambda x: 1 - sum(x)}) # values summing up to 1 + r = optimize.minimize(match, x0=uniform_distribution, method='SLSQP', bounds=bounds, constraints=constraints) + return r.x + + + +class KDExML(BaseQuantifier, KDEBase): + + def __init__(self, bandwidth=0.1, standardize=False): + self._check_bandwidth(bandwidth) + self.bandwidth = bandwidth + self.standardize = standardize + + def fit(self, X, y): + classes = sorted(np.unique(y)) + + if self.standardize: + self.scaler = StandardScaler() + X = self.scaler.fit_transform(X) + + if issparse(X): + X = X.toarray() + + self.mix_densities = self.get_mixture_components(X, y, classes, self.bandwidth) + return self + + def predict(self, X): + """ + Searches for the mixture model parameter (the sought prevalence values) that maximizes the likelihood + of the data (i.e., that minimizes the negative log-likelihood) + + :param X: instances in the sample + :return: a vector of class prevalence estimates + """ + epsilon = 1e-10 + if issparse(X): + X = X.toarray() + n_classes = len(self.mix_densities) + if self.standardize: + X = self.scaler.transform(X) + test_densities = [self.pdf(kde_i, X) for kde_i in self.mix_densities] + + def neg_loglikelihood(prev): + test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) + test_loglikelihood = np.log(test_mixture_likelihood + epsilon) + return -np.sum(test_loglikelihood) + + return F.optim_minimize(neg_loglikelihood, n_classes) + + + +if __name__ == '__main__': + + qp.environ['SAMPLE_SIZE'] = 250 + qp.environ['N_JOBS'] = -1 + min_df = 10 + # dataset = 'imdb' + repeats = 10 + error = 'mae' + + div = 'topsoe' + + # generates tuples (dataset, method, method_name) + # (the dataset is needed for methods that process the dataset differently) + def gen_methods(): + + for dataset in qp.datasets.REVIEWS_SENTIMENT_DATASETS: + + data = qp.datasets.fetch_reviews(dataset, tfidf=False) + + # bernoulli_vectorizer = CountVectorizer(min_df=min_df, binary=True) + # dxs = DxS(divergence=div, vectorizer=bernoulli_vectorizer) + # yield data, dxs, 'DxS-Bernoulli' + # + # multinomial_vectorizer = CountVectorizer(min_df=min_df, binary=False) + # dxs = DxS(divergence=div, vectorizer=multinomial_vectorizer) + # yield data, dxs, 'DxS-multinomial' + # + # tf_vectorizer = TfidfVectorizer(sublinear_tf=False, use_idf=False, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=tf_vectorizer) + # yield data, dxs, 'DxS-TF' + # + # logtf_vectorizer = TfidfVectorizer(sublinear_tf=True, use_idf=False, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=logtf_vectorizer) + # yield data, dxs, 'DxS-logTF' + # + # tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm=None) + # dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + # yield data, dxs, 'DxS-TFIDF' + # + # tfidf_vectorizer = TfidfVectorizer(use_idf=True, min_df=min_df, norm='l2') + # dxs = DxS(divergence=div, vectorizer=tfidf_vectorizer) + # yield data, dxs, 'DxS-TFIDF-l2' + + tsr_vectorizer = TSRweighting(tsr_function=information_gain, min_df=min_df, norm='l2') + dxs = DxS(divergence=div, vectorizer=tsr_vectorizer) + yield data, dxs, 'DxS-TFTSR-l2' + + data = qp.datasets.fetch_reviews(dataset, tfidf=True, min_df=min_df) + + kdex = KDExML() + reduction = TruncatedSVD(n_components=100, random_state=0) + red_data = qp.data.preprocessing.instance_transformation(data, transformer=reduction, inplace=False) + yield red_data, kdex, 'KDEx' + + hdy = HDy(LogisticRegression()) + yield data, hdy, 'HDy' + + # dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=5) + # yield data, dm, 'DM-5b' + # + # dm = DistributionMatchingY(LogisticRegression(), divergence=div, nbins=10) + # yield data, dm, 'DM-10b' + + + + + result_path = 'results.csv' + with open(result_path, 'wt') as csv: + csv.write(f'Method\tDataset\tMAE\tMRAE\n') + for data, quantifier, quant_name in gen_methods(): + quantifier.fit(*data.training.Xy) + report = qp.evaluation.evaluation_report(quantifier, APP(data.test, repeats=repeats), error_metrics=['mae','mrae'], verbose=True) + means = report.mean(numeric_only=True) + csv.write(f'{quant_name}\t{data.name}\t{means["mae"]:.5f}\t{means["mrae"]:.5f}\n') + + df = pd.read_csv(result_path, sep='\t') + # print(df) + + pv = df.pivot_table(index='Method', columns="Dataset", values=["MAE", "MRAE"]) + print(pv) + + + + diff --git a/quapy/__init__.py b/quapy/__init__.py index d013f5b..182c39a 100644 --- a/quapy/__init__.py +++ b/quapy/__init__.py @@ -7,13 +7,17 @@ from . import functional from . import method from . import evaluation from . import protocol -from . import plot from . import util from . import model_selection from . import classification import os -__version__ = '0.2.0' +try: + from . import plot +except ImportError: + plot = None + +__version__ = '0.2.1' def _default_cls(): @@ -74,4 +78,3 @@ def _get_classifier(classifier): raise ValueError('neither classifier nor qp.environ["DEFAULT_CLS"] have been specified') return classifier - diff --git a/quapy/classification/__init__.py b/quapy/classification/__init__.py index 7aa5c77..b63e0eb 100644 --- a/quapy/classification/__init__.py +++ b/quapy/classification/__init__.py @@ -1 +1,3 @@ -from . import svmperf \ No newline at end of file +from . import calibration +from . import methods +from . import svmperf diff --git a/quapy/classification/calibration.py b/quapy/classification/calibration.py index 0f5e9f7..04ce2f9 100644 --- a/quapy/classification/calibration.py +++ b/quapy/classification/calibration.py @@ -1,8 +1,9 @@ from copy import deepcopy -from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling from sklearn.base import BaseEstimator, clone from sklearn.model_selection import cross_val_predict, train_test_split +from sklearn.preprocessing import LabelEncoder +from sklearn.utils.validation import check_X_y import numpy as np @@ -11,6 +12,17 @@ import numpy as np # see https://github.com/kundajelab/abstention +def _require_abstention_calibration(): + try: + from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling + except ImportError as exc: + raise ImportError( + "Calibration methods in quapy.classification.calibration require the optional " + "'abstention' package." + ) from exc + return NoBiasVectorScaling, TempScaling, VectorScaling + + class RecalibratedProbabilisticClassifier: """ Abstract class for (re)calibration method from `abstention.calibration`, as defined in @@ -142,6 +154,7 @@ class NBVSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + NoBiasVectorScaling, _, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = NoBiasVectorScaling(verbose=verbose) self.val_split = val_split @@ -164,6 +177,7 @@ class BCTSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, TempScaling, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = TempScaling(verbose=verbose, bias_positions='all') self.val_split = val_split @@ -186,6 +200,7 @@ class TSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, TempScaling, _ = _require_abstention_calibration() self.classifier = classifier self.calibrator = TempScaling(verbose=verbose) self.val_split = val_split @@ -208,9 +223,84 @@ class VSCalibration(RecalibratedProbabilisticClassifierBase): """ def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False): + _, _, VectorScaling = _require_abstention_calibration() self.classifier = classifier self.calibrator = VectorScaling(verbose=verbose) self.val_split = val_split self.n_jobs = n_jobs self.verbose = verbose + +class TemperatureScalingFromLogits(BaseEstimator): + """ + Calibrates a matrix of logits by learning a temperature-scaling mapping + with the calibration methods from `abstention.calibration`. + + This estimator is useful when the inputs are already logits produced by a + pretrained classifier, and the goal is to transform them directly into + calibrated posterior probabilities without retraining the underlying model. + + :param bias_corrected: if True, uses Bias-Corrected Temperature Scaling + (BCTS); otherwise, uses standard Temperature Scaling (TS) + :param verbose: whether the underlying calibrator should display progress + information + """ + + def __init__(self, bias_corrected=False, verbose=False): + self.bias_corrected = bias_corrected + self.verbose = verbose + + def fit(self, X, y): + """ + Fits the logits calibrator. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :param y: array-like of shape `(n_samples,)` containing class labels + :return: self + """ + X, y = check_X_y(X, y) + + self.label_encoder_ = LabelEncoder() + y_enc = self.label_encoder_.fit_transform(y) + self.classes_ = self.label_encoder_.classes_ + + n_classes = len(self.classes_) + logits_dim = X.shape[1] + if n_classes != logits_dim: + raise ValueError( + f'mismatch between the number of classes ({n_classes}) and the ' + f'dimensionality of the logits ({logits_dim})' + ) + + _, TempScaling, _ = _require_abstention_calibration() + calibrator = TempScaling( + verbose=self.verbose, + bias_positions='all' if self.bias_corrected else [], + ) + self.calibrator_ = calibrator + self.calibration_function_ = calibrator(X, np.eye(n_classes)[y_enc]) + return self + + def predict_proba(self, X): + """ + Converts logits into calibrated posterior probabilities. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :return: array-like of shape `(n_samples, n_classes)` with calibrated + posterior probabilities + """ + return self.calibration_function_(X) + + def predict(self, X): + """ + Predicts class labels after calibration. + + :param X: array-like of shape `(n_samples, n_classes)` containing + logits + :return: array-like of shape `(n_samples,)` with class label + predictions + """ + posteriors = self.predict_proba(X) + return self.label_encoder_.inverse_transform(np.argmax(posteriors, axis=1)) diff --git a/quapy/classification/methods.py b/quapy/classification/methods.py index 85300d0..1876c4a 100644 --- a/quapy/classification/methods.py +++ b/quapy/classification/methods.py @@ -1,3 +1,4 @@ +import numpy as np from sklearn.base import BaseEstimator from sklearn.decomposition import TruncatedSVD from sklearn.linear_model import LogisticRegression @@ -95,3 +96,23 @@ class LowRankLogisticRegression(BaseEstimator): if self.pca is None: return X return self.pca.transform(X) + + +class MockClassifierFromPosteriors(BaseEstimator): + """ + Mock classifier that bypasses classifier training when the input instances + are already posterior probabilities produced by a pretrained probabilistic + classifier. + + :param X: arrays of shape `(n_samples, n_classes)` are interpreted as posterior probabilities + """ + + def fit(self, X, y): + self.classes_ = np.sort(np.unique(y)) + return self + + def predict(self, X): + return np.argmax(X, axis=1) + + def predict_proba(self, X): + return X diff --git a/quapy/classification/neural.py b/quapy/classification/neural.py index 8d78d7c..4cdd0d6 100644 --- a/quapy/classification/neural.py +++ b/quapy/classification/neural.py @@ -1,3 +1,4 @@ +import logging import os from abc import ABCMeta, abstractmethod from pathlib import Path @@ -42,7 +43,7 @@ class NeuralClassifierTrainer: batch_size=64, batch_size_test=512, padding_length=300, - device='cuda', + device='cpu', checkpointpath='../checkpoint/classifier_net.dat'): super().__init__() @@ -63,7 +64,7 @@ class NeuralClassifierTrainer: self.learner_hyperparams = self.net.get_params() self.checkpointpath = checkpointpath - print(f'[NeuralNetwork running on {device}]') + logging.getLogger(__name__).info(f'NeuralNetwork running on {device}') os.makedirs(Path(checkpointpath).parent, exist_ok=True) def reset_net_params(self, vocab_size, n_classes): @@ -198,14 +199,15 @@ class NeuralClassifierTrainer: if self.early_stop.IMPROVED: torch.save(self.net.state_dict(), checkpoint) elif self.early_stop.STOP: - print(f'training ended by patience exhasted; loading best model parameters in {checkpoint} ' - f'for epoch {self.early_stop.best_epoch}') + logging.getLogger(__name__).info( + f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' + f'for epoch {self.early_stop.best_epoch}') self.net.load_state_dict(torch.load(checkpoint)) break - print('performing one training pass over the validation set...') + logging.getLogger(__name__).info('performing one training pass over the validation set...') self._train_epoch(valid_generator, self.status['tr'], pbar, epoch=0) - print('[done]') + logging.getLogger(__name__).info('done') return self diff --git a/quapy/classification/svmperf.py b/quapy/classification/svmperf.py index 71f2ac3..b7ba007 100644 --- a/quapy/classification/svmperf.py +++ b/quapy/classification/svmperf.py @@ -1,3 +1,4 @@ +import logging import random import shutil import subprocess @@ -66,8 +67,7 @@ class SVMperf(BaseEstimator, ClassifierMixin): # this would allow to run parallel instances of predict random_code = 'svmperfprocess'+'-'.join(str(local_random.randint(0, 1000000)) for _ in range(5)) if self.host_folder is None: - # tmp dir are removed after the fit terminates in multiprocessing... - self.tmpdir = tempfile.TemporaryDirectory(suffix=random_code).name + self.tmpdir = join(tempfile.gettempdir(), random_code) else: self.tmpdir = join(self.host_folder, '.' + random_code) makedirs(self.tmpdir, exist_ok=True) @@ -79,14 +79,14 @@ class SVMperf(BaseEstimator, ClassifierMixin): cmd = ' '.join([self.svmperf_learn, self.c_cmd, self.loss_cmd, traindat, self.model]) if self.verbose: - print('[Running]', cmd) - p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT) + logging.getLogger(__name__).info(f'[Running] {cmd}') + p = subprocess.run(cmd.split(), stdout=PIPE, stderr=PIPE) if not exists(self.model): - print(p.stderr.decode('utf-8')) + logging.getLogger(__name__).error(p.stderr.decode('utf-8')) remove(traindat) if self.verbose: - print(p.stdout.decode('utf-8')) + logging.getLogger(__name__).info(p.stdout.decode('utf-8')) return self @@ -125,11 +125,11 @@ class SVMperf(BaseEstimator, ClassifierMixin): cmd = ' '.join([self.svmperf_classify, testdat, self.model, predictions_path]) if self.verbose: - print('[Running]', cmd) + logging.getLogger(__name__).info(f'[Running] {cmd}') p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT) if self.verbose: - print(p.stdout.decode('utf-8')) + logging.getLogger(__name__).info(p.stdout.decode('utf-8')) scores = np.loadtxt(predictions_path) remove(testdat) diff --git a/quapy/data/_lequa.py b/quapy/data/_lequa.py index e162f4c..93299d4 100644 --- a/quapy/data/_lequa.py +++ b/quapy/data/_lequa.py @@ -99,6 +99,9 @@ class SamplesFromDir(AbstractProtocol): sample, _ = self.load_fn(os.path.join(self.path_dir, f'{id}.txt')) yield sample, prevalence + def total(self): + return len(self.true_prevs) + class LabelledCollectionsFromDir(AbstractProtocol): @@ -113,6 +116,10 @@ class LabelledCollectionsFromDir(AbstractProtocol): lc = LabelledCollection.load(path=collection_path, loader_func=self.load_fn) yield lc + def total(self): + return len(self.true_prevs) + + class ResultSubmission: @@ -180,8 +187,7 @@ class ResultSubmission: try: df = pd.read_csv(path, index_col=0) except Exception as e: - print(f'the file {path} does not seem to be a valid csv file. ') - print(e) + raise ValueError(f'the file {path} does not seem to be a valid csv file: {e}') return ResultSubmission.check_dataframe_format(df, path=path) @classmethod diff --git a/quapy/data/base.py b/quapy/data/base.py index 82c57db..934d0f0 100644 --- a/quapy/data/base.py +++ b/quapy/data/base.py @@ -33,7 +33,6 @@ class LabelledCollection: else: self.instances = np.asarray(instances) self.labels = np.asarray(labels) - n_docs = len(self) if classes is None: self.classes_ = F.classes_from_labels(self.labels) else: @@ -41,7 +40,13 @@ class LabelledCollection: self.classes_.sort() if len(set(self.labels).difference(set(classes))) > 0: raise ValueError(f'labels ({set(self.labels)}) contain values not included in classes_ ({set(classes)})') - self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_} + self._index = None + + @property + def index(self): + if not hasattr(self, '_index') or self._index is None: + self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_} + return self._index @classmethod def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs): @@ -324,7 +329,9 @@ class LabelledCollection: else: raise NotImplementedError('unsupported operation for collection types') labels = np.concatenate([lc.labels for lc in args]) - classes = np.unique(labels).sort() + # union of each collection's own classes_, so a class declared but absent from + # this particular join (e.g. an empty fold) is preserved at zero prevalence + classes = np.unique(np.concatenate([lc.classes_ for lc in args])) return LabelledCollection(instances, labels, classes=classes) @property diff --git a/quapy/data/datasets.py b/quapy/data/datasets.py index c08748f..a1d5d54 100644 --- a/quapy/data/datasets.py +++ b/quapy/data/datasets.py @@ -1,9 +1,9 @@ +import logging import os from contextlib import contextmanager import zipfile from os.path import join import pandas as pd -from ucimlrepo import fetch_ucirepo from quapy.data.base import Dataset, LabelledCollection from quapy.data.preprocessing import text2tfidf, reduce_columns from quapy.data.preprocessing import standardize as standardizer @@ -12,6 +12,17 @@ from quapy.util import download_file_if_not_exists, download_file, get_quapy_hom from sklearn.preprocessing import StandardScaler +def _fetch_ucirepo(*args, **kwargs): + try: + from ucimlrepo import fetch_ucirepo + except ImportError as exc: + raise ImportError( + "UCI dataset fetching requires the optional 'ucimlrepo' package. " + "Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset." + ) from exc + return fetch_ucirepo(*args, **kwargs) + + REVIEWS_SENTIMENT_DATASETS = ['hp', 'kindle', 'imdb'] TWITTER_SENTIMENT_DATASETS_TEST = [ @@ -109,6 +120,9 @@ LEQUA2024_SAMPLE_SIZE = { 'T4': 250, } +IMAGE_DATASETS=['cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'] +IMAGE_EMBEDDINGS=['features', 'logits', 'predictions'] + def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset: """ @@ -199,8 +213,9 @@ def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_hom if dataset_name in {'semeval13', 'semeval14', 'semeval15'}: trainset_name = 'semeval' testset_name = 'semeval' if for_model_selection else dataset_name - print(f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common " - f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}") + logging.getLogger(__name__).info( + f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common " + f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}") else: if dataset_name == 'semeval' and for_model_selection==False: raise ValueError('dataset "semeval" can only be used for model selection. ' @@ -500,11 +515,11 @@ def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize= y = df["NSP"].astype(int).values elif group == "semeion": with download_tmp_file("semeion", "semeion.data") as tmp: - df = pd.read_csv(tmp, header=None, sep='\s+') + df = pd.read_csv(tmp, header=None, sep='\\s+') X = df.iloc[:, 0:256].astype(float).values y = df[263].values # 263 stands for digit 8 (labels are one-hot vectors from col 256-266) else: - df = fetch_ucirepo(id=id) + df = _fetch_ucirepo(id=id) X, y = df.data.features.to_numpy(), df.data.targets.to_numpy().squeeze() # transform data when needed before returning (returned data will be pickled) @@ -616,8 +631,8 @@ def fetch_UCIMulticlassDataset( are taken for training, and the rest (irrespective of `min_test_split`) is taken for test. :param max_train_instances: maximum number of instances to keep for training (defaults to 25000); set to -1 or None to avoid this check - :param min_class_support: minimum number of istances per class. Classes with fewer instances - are discarded (deafult is 100) + :param min_class_support: integer or float, the minimum number or proportion of istances per class. + Classes with fewer instances are discarded (deafult is 100). :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested, standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the training portion of the data. @@ -673,6 +688,11 @@ def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_clas f'Name {dataset_name} does not match any known dataset from the ' \ f'UCI Machine Learning datasets repository (multiclass). ' \ f'Valid ones are {UCI_MULTICLASS_DATASETS}' + + assert (min_class_support is None or + ((isinstance(min_class_support, int) and min_class_support >= 0) or + (isinstance(min_class_support, float) and 0. <= min_class_support < 1.))), \ + f'invalid value for {min_class_support=}; expected non negative integer or float in [0,1)' if data_home is None: data_home = get_quapy_home() @@ -739,26 +759,43 @@ def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_clas file = join(data_home, 'uci_multiclass', dataset_name+'.pkl') + def dummify_categorical_features(df_features, dataset_id): + categorical_features = { + 158: ["S1", "C1", "S2", "C2", "S3", "C3", "S4", "C4", "S5", "C5"], # poker_hand + } + + categorical = categorical_features.get(dataset_id, []) + + X = df_features.copy() + if categorical: + X[categorical] = X[categorical].astype("category") + X = pd.get_dummies(X, columns=categorical, drop_first=True) + + return X + def download(id, name): - df = fetch_ucirepo(id=id) + df = _fetch_ucirepo(id=id) - df.data.features = pd.get_dummies(df.data.features, drop_first=True) - X, y = df.data.features.to_numpy(dtype=np.float64), df.data.targets.to_numpy().squeeze() + X_df = dummify_categorical_features(df.data.features, id) + X = X_df.to_numpy(dtype=np.float64) + y = df.data.targets.to_numpy().squeeze() - assert y.ndim == 1, 'more than one y' + assert y.ndim == 1, f'error: the dataset {id=} {name=} has more than one target variable' classes = np.sort(np.unique(y)) y = np.searchsorted(classes, y) return LabelledCollection(X, y) - def filter_classes(data: LabelledCollection, min_ipc): - if min_ipc is None: - min_ipc = 0 + def filter_classes(data: LabelledCollection, min_class_support): + if min_class_support is None or min_class_support == 0.: + return data + if isinstance(min_class_support, float): + min_class_support = int(len(data) * min_class_support) classes = data.classes_ - # restrict classes to only those with at least min_ipc instances - classes = classes[data.counts() >= min_ipc] + # restrict classes to only those with at least min_class_support instances + classes = classes[data.counts() >= min_class_support] # filter X and y keeping only datapoints belonging to valid classes - filter_idx = np.in1d(data.y, classes) + filter_idx = np.isin(data.y, classes) X, y = data.X[filter_idx], data.y[filter_idx] # map classes to range(len(classes)) y = np.searchsorted(classes, y) @@ -1030,3 +1067,90 @@ def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=No return train, test_gen else: return train_gen, test_gen + + +def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]: + """ + Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo `_. + + Embeddings were extracted using `this script `_. + + :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist' + :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation values), 'predictions' (posterior probabilities) + :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default + ~/quay_data/ directory) + :return: a tuple (train, val, test) where each entry is an instance of :class:`quapy.data.base.LabelledCollection` + """ + assert dataset_name in IMAGE_DATASETS, \ + f'Name {dataset_name} does not match any known dataset. Valid ones are {IMAGE_DATASETS}' + assert embedding in IMAGE_EMBEDDINGS, \ + f'Name {embedding} does not match any known type of embedding. Valid ones are {IMAGE_EMBEDDINGS}' + if data_home is None: + data_home = get_quapy_home() + + dataset_network = { + 'cifar10': 'resnet18', + 'cifar100': 'resnet18', + 'cifar100coarse': 'resnet18', + 'svhn': 'resnet18', + 'fashionmnist': 'basiccnn', + 'mnist': 'basiccnn', + } + + trained_network = dataset_network[dataset_name] + + def download_embedding_npz(dataset_name, trained_network, embedding): + target_file = f'{dataset_name}_{trained_network}_{embedding}.npz' + URL = f'https://zenodo.org/records/21131944/files/{target_file}' + os.makedirs(join(data_home, 'image'), exist_ok=True) + file_path = join(data_home, 'image', target_file) + download_file_if_not_exists(URL, file_path) + npz_file = np.load(file_path) + return npz_file + + embedding_dict = download_embedding_npz(dataset_name, trained_network, embedding=embedding) + labels_dict = download_embedding_npz(dataset_name, trained_network, embedding='targets') + + train = LabelledCollection(embedding_dict['train'], labels_dict['train']) + val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes) + test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes) + + return train, val, test + + +def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset: + """ + Loads an image dataset with pre-generated embeddings. Available datasets include: + + - 'cifar10', 'cifar100', 'cifar100coarse': see `Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, Toronto, Ontario, 2009. `_ + - 'mnist': `Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998. `_ + - 'fashionmnist': `Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017. `_ + - 'svhn': `Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y Ng, et al. Reading digits in natural images with unsupervised feature learning. In NIPS workshop on deep learning and unsupervised feature learning, volume 2011, page 4. Granada, 2011. `_ + + The image dataset are stored in `Zenodo `_ and were extracted using `this script `_. + + These embeddings were generated using a resnet18 or a simple cnn. In all cases, the network was trained using ~60% of the data, validated on ~25% of the data, and the remaining ~15% was used for test. Splits were created with stratification. + Once the network is trained, it was used with frozen weights to generate embeddings for the training, validation, and test, in different formats (see below). + It would therefore be convenient to use only heldout data (validation and test) for training and testing quantifiers (this is the default behavior), although the training+validation data can be accessed with `heldout_only=False`. + + :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist' + :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation outputs), 'predictions' (post-softmax outputs, or predicted posterior probabilities) + :param heldout_only: whether to discard the part of the training data used to train the neural model that generated the embeddings (default: True); set to False + to obtain, as the training data, the original training+validation splits. + :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default + ~/quay_data/ directory) + :return: an instance of :class:`quapy.data.base.Dataset` + """ + if data_home is None: + data_home = get_quapy_home() + + network_train, val, test = _fetch_image_embedding_splits(dataset_name, embedding, data_home) + + if heldout_only: + train = val + else: + train = network_train + val + + return Dataset(train, test, name=dataset_name) + + diff --git a/quapy/data/preprocessing.py b/quapy/data/preprocessing.py index a4be4fd..5f7e0a9 100644 --- a/quapy/data/preprocessing.py +++ b/quapy/data/preprocessing.py @@ -10,6 +10,37 @@ from quapy.util import map_parallel from .base import LabelledCollection +def instance_transformation(dataset:Dataset, transformer, inplace=False): + """ + Transforms a :class:`quapy.data.base.Dataset` applying the `fit_transform` and `transform` functions + of a (sklearn's) transformer. + + :param dataset: a :class:`quapy.data.base.Dataset` where the instances of training and test collections are + lists of str + :param transformer: TransformerMixin implementing `fit_transform` and `transform` functions + :param inplace: whether or not to apply the transformation inplace (True), or to a new copy (False, default) + :return: a new :class:`quapy.data.base.Dataset` with transformed instances (if inplace=False) or a reference to the + current Dataset (if inplace=True) where the instances have been transformed + """ + training_transformed = transformer.fit_transform(*dataset.training.Xy) + test_transformed = transformer.transform(dataset.test.X) + orig_name = dataset.name + + if inplace: + dataset.training = LabelledCollection(training_transformed, dataset.training.labels, dataset.classes_) + dataset.test = LabelledCollection(test_transformed, dataset.test.labels, dataset.classes_) + if hasattr(transformer, 'vocabulary_'): + dataset.vocabulary = transformer.vocabulary_ + return dataset + else: + training = LabelledCollection(training_transformed, dataset.training.labels.copy(), dataset.classes_) + test = LabelledCollection(test_transformed, dataset.test.labels.copy(), dataset.classes_) + vocab = None + if hasattr(transformer, 'vocabulary_'): + vocab = transformer.vocabulary_ + return Dataset(training, test, vocabulary=vocab, name=orig_name) + + def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kwargs): """ Transforms a :class:`quapy.data.base.Dataset` of textual instances into a :class:`quapy.data.base.Dataset` of @@ -29,18 +60,7 @@ def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kw __check_type(dataset.test.instances, np.ndarray, str) vectorizer = TfidfVectorizer(min_df=min_df, sublinear_tf=sublinear_tf, **kwargs) - training_documents = vectorizer.fit_transform(dataset.training.instances) - test_documents = vectorizer.transform(dataset.test.instances) - - if inplace: - dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.classes_) - dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.classes_) - dataset.vocabulary = vectorizer.vocabulary_ - return dataset - else: - training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.classes_) - test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.classes_) - return Dataset(training, test, vectorizer.vocabulary_) + return instance_transformation(dataset, vectorizer, inplace) def reduce_columns(dataset: Dataset, min_df=5, inplace=False): diff --git a/quapy/data/reader.py b/quapy/data/reader.py index 88791e3..5c95b49 100644 --- a/quapy/data/reader.py +++ b/quapy/data/reader.py @@ -1,3 +1,5 @@ +import logging + import numpy as np from scipy.sparse import dok_matrix from tqdm import tqdm @@ -30,7 +32,7 @@ def from_text(path, encoding='utf-8', verbose=1, class2int=True): all_sentences.append(sentence) all_labels.append(label) except ValueError: - print(f'format error in {line}') + logging.getLogger(__name__).warning(f'format error in {line}') return all_sentences, all_labels diff --git a/quapy/error.py b/quapy/error.py index eb42cd6..0466b94 100644 --- a/quapy/error.py +++ b/quapy/error.py @@ -3,6 +3,7 @@ import numpy as np from sklearn.metrics import f1_score import quapy as qp +from quapy.functional import AitchisonDistance def from_name(err_name): @@ -128,6 +129,82 @@ def se(prevs_true, prevs_hat): return ((prevs_hat - prevs_true) ** 2).mean(axis=-1) +def sre(prevs_true, prevs_hat, prevs_train, eps=0.): + """ + Computes the squared ratio error between two prevalence vectors. + The squared ratio error between prevalence vectors :math:`p` and + :math:`\\hat{p}` with training prevalence :math:`p^{tr}` is: + :math:`SRE(p,\\hat{p},p^{tr})=\\frac{1}{|\\mathcal{Y}|}\\sum_{i \\in \\mathcal{Y}}(w_i-\\hat{w}_i)^2`, + where :math:`w_i=\\frac{p_i}{p^{tr}_i}`. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :param prevs_train: array-like with the training prevalence values, or a single + prevalence vector when all comparisons refer to the same training set + :param eps: smoothing factor for the prevalence values (default 0, i.e., no smoothing) + :return: squared ratio error + """ + prevs_true = np.asarray(prevs_true) + prevs_hat = np.asarray(prevs_hat) + prevs_train = np.asarray(prevs_train) + assert prevs_true.shape == prevs_hat.shape, f'wrong shape {prevs_true.shape=} vs {prevs_hat.shape=}' + assert prevs_true.shape[-1] == prevs_train.shape[-1], 'wrong shape for training prevalence' + if prevs_true.ndim == 2 and prevs_train.ndim == 1: + prevs_train = np.tile(prevs_train, reps=(prevs_true.shape[0], 1)) + if eps > 0: + prevs_true = smooth(prevs_true, eps) + prevs_hat = smooth(prevs_hat, eps) + prevs_train = smooth(prevs_train, eps) + + n_classes = prevs_true.shape[-1] + w = prevs_true / prevs_train + w_hat = prevs_hat / prevs_train + return (1. / n_classes) * np.sum((w - w_hat) ** 2., axis=-1) + + +def msre(prevs_true, prevs_hat, prevs_train, eps=0.): + """ + Computes the mean squared ratio error (see :meth:`quapy.error.sre`) across the sample pairs. + + :param prevs_true: array-like of shape `(n_samples, n_classes,)` with the true prevalence values + :param prevs_hat: array-like of shape equal to prevs_true with the predicted prevalence values + :param prevs_train: array-like with the training prevalence values + :param eps: smoothing factor (default 0, i.e., no smoothing) + :return: mean squared ratio error + """ + return np.mean(sre(prevs_true, prevs_hat, prevs_train, eps)) + + +def aqe(prevs_true, prevs_hat): + """ + Computes the Aitchison distance between two prevalence vectors. + The Aitchison distance between prevalence vectors :math:`p` and + :math:`\\hat{p}` is computed as + :math:`d_A(p,\\hat{p})=\\|\\mathrm{clr}(p)-\\mathrm{clr}(\\hat{p})\\|_2`, + where :math:`\\mathrm{clr}(p)_i=\\log p_i-\\frac{1}{|\\mathcal{Y}|} + \\sum_{j \\in \\mathcal{Y}} \\log p_j`. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :return: Aitchison distance + """ + return AitchisonDistance(prevs_true, prevs_hat) + + +def maqe(prevs_true, prevs_hat): + """ + Computes the mean Aitchison distance (see :meth:`quapy.error.aitchisondist`) + across the sample pairs, i.e., + :math:`\\mathrm{mAitchisonDist}=\\frac{1}{n}\\sum_{i=1}^n + d_A(p_i,\\hat{p}_i)`. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :return: mean Aitchison distance + """ + return np.mean(aqe(prevs_true, prevs_hat)) + + def mkld(prevs_true, prevs_hat, eps=None): """Computes the mean Kullback-Leibler divergence (see :meth:`quapy.error.kld`) across the sample pairs. The distributions are smoothed using the `eps` factor @@ -374,8 +451,8 @@ def __check_eps(eps=None): CLASSIFICATION_ERROR = {f1e, acce} -QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld} -QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld} +QUANTIFICATION_ERROR = {mae, mnae, mrae, mnrae, mse, mkld, mnkld, msre, maqe} +QUANTIFICATION_ERROR_SINGLE = {ae, nae, rae, nrae, se, kld, nkld, sre, aqe} QUANTIFICATION_ERROR_SMOOTH = {kld, nkld, rae, nrae, mkld, mnkld, mrae} CLASSIFICATION_ERROR_NAMES = {func.__name__ for func in CLASSIFICATION_ERROR} QUANTIFICATION_ERROR_NAMES = {func.__name__ for func in QUANTIFICATION_ERROR} @@ -387,6 +464,9 @@ ERROR_NAMES = \ f1_error = f1e acc_error = acce mean_absolute_error = mae +squared_ratio_error = sre +dist_aitchison = aqe +mean_dist_aitchison = maqe absolute_error = ae mean_relative_absolute_error = mrae relative_absolute_error = rae diff --git a/quapy/functional.py b/quapy/functional.py index 408c62a..392309e 100644 --- a/quapy/functional.py +++ b/quapy/functional.py @@ -1,11 +1,15 @@ import warnings +from abc import ABC, abstractmethod from collections import defaultdict +from functools import lru_cache from typing import Literal, Union, Callable from numpy.typing import ArrayLike import scipy import numpy as np +import quapy as qp + # ------------------------------------------------------------------------------------------ # General utils @@ -277,7 +281,7 @@ def l1_norm(prevalences: ArrayLike) -> np.ndarray: """ n_classes = prevalences.shape[-1] accum = prevalences.sum(axis=-1, keepdims=True) - prevalences = np.true_divide(prevalences, accum, where=accum > 0) + prevalences = np.true_divide(prevalences, accum, where=accum > 0, out=None) allzeros = accum.flatten() == 0 if any(allzeros): if prevalences.ndim == 1: @@ -389,6 +393,23 @@ def TopsoeDistance(P: np.ndarray, Q: np.ndarray, epsilon: float=1e-20): return np.sum(P*np.log((2*P+epsilon)/(P+Q+epsilon)) + Q*np.log((2*Q+epsilon)/(P+Q+epsilon))) +def AitchisonDistance(prevs_true, prevs_hat): + """ + Computes the Aitchison distance between two prevalence vectors. + The Aitchison distance between prevalence vectors :math:`p` and + :math:`\\hat{p}` is computed as + :math:`d_A(p,\\hat{p})=\\|\\mathrm{clr}(p)-\\mathrm{clr}(\\hat{p})\\|_2`, + where :math:`\\mathrm{clr}(p)_i=\\log p_i-\\frac{1}{|\\mathcal{Y}|} + \\sum_{j \\in \\mathcal{Y}} \\log p_j`. + + :param prevs_true: array-like with the true prevalence values + :param prevs_hat: array-like with the predicted prevalence values + :return: Aitchison distance + """ + clr = CLRtransformation() + return np.linalg.norm(clr(prevs_true) - clr(prevs_hat), axis=-1) + + def get_divergence(divergence: Union[str, Callable]): """ Guarantees that the divergence received as argument is a function. That is, if this argument is already @@ -403,6 +424,8 @@ def get_divergence(divergence: Union[str, Callable]): return HellingerDistance elif divergence=='topsoe': return TopsoeDistance + elif divergence=='aitchison': + return AitchisonDistance else: raise ValueError(f'unknown divergence {divergence}') elif callable(divergence): @@ -426,7 +449,7 @@ def argmin_prevalence(loss: Callable, :param method: string indicating the search strategy. Possible values are:: 'optim_minimize': uses scipy.optim 'linear_search': carries out a linear search for binary problems in the space [0, 0.01, 0.02, ..., 1] - 'ternary_search': implements the ternary search (not yet implemented) + 'ternary_search': carries out a ternary search for binary problems in the interval [0,1] :return: np.ndarray, a prevalence vector """ if method == 'optim_minimize': @@ -434,7 +457,7 @@ def argmin_prevalence(loss: Callable, elif method == 'linear_search': return linear_search(loss, n_classes) elif method == 'ternary_search': - ternary_search(loss, n_classes) + return ternary_search(loss, n_classes) else: raise NotImplementedError() @@ -489,7 +512,32 @@ def linear_search(loss: Callable, n_classes: int): def ternary_search(loss: Callable, n_classes: int): - raise NotImplementedError() + """ + Performs a ternary search for the best prevalence value in binary problems. + This search assumes the loss is unimodal over the interval [0,1]. + + :param loss: (callable) the function to minimize + :param n_classes: (int) the number of classes, i.e., the dimensionality of the prevalence vector + :return: (ndarray) the best prevalence vector found + """ + assert n_classes == 2, 'ternary search is only available for binary problems' + + left, right = 0., 1. + tol = 1e-5 + while abs(right - left) >= tol: + left_third = left + (right - left) / 3 + right_third = right - (right - left) / 3 + + left_loss = loss(np.asarray([1 - left_third, left_third])) + right_loss = loss(np.asarray([1 - right_third, right_third])) + + if left_loss < right_loss: + right = right_third + else: + left = left_third + + prev = (left + right) / 2 + return np.asarray([1 - prev, prev]) # ------------------------------------------------------------------------------------------ @@ -621,7 +669,11 @@ def solve_adjustment( if method == "inversion": pass # We leave A and B unchanged elif method == "invariant-ratio": - # Change the last equation to replace it with the normalization condition + # Change the last equation to replace it with the normalization condition; + # copy first so this does not mutate the caller's arrays (np.asarray above + # returns the same object, not a copy, when the input is already float64) + A = A.copy() + B = B.copy() A[-1, :] = 1.0 B[-1] = 1.0 else: @@ -649,3 +701,105 @@ def solve_adjustment( raise ValueError(f'unknown {solver=}') +# ------------------------------------------------------------------------------------------ +# Transformations from Compositional analysis +# ------------------------------------------------------------------------------------------ + +class CompositionalTransformation(ABC): + """ + Abstract class of transformations for compositional data. + """ + + EPSILON = 1e-12 + + @abstractmethod + def __call__(self, X): + ... + + @abstractmethod + def inverse(self, Z): + ... + + +class CLRtransformation(CompositionalTransformation): + """ + Centered log-ratio (CLR) transformation. + """ + + def __call__(self, X): + X = np.asarray(X) + X = qp.error.smooth(X, self.EPSILON) + geometric_mean = np.exp(np.mean(np.log(X), axis=-1, keepdims=True)) + return np.log(X / geometric_mean) + + def inverse(self, Z): + return scipy.special.softmax(Z, axis=-1) + + +class ILRtransformation(CompositionalTransformation): + """ + Isometric log-ratio (ILR) transformation. + """ + + def __call__(self, X): + X = np.asarray(X) + X = qp.error.smooth(X, self.EPSILON) + basis = self.get_V(X.shape[-1]) + return np.log(X) @ basis.T + + def inverse(self, Z): + Z = np.asarray(Z) + basis = self.get_V(Z.shape[-1] + 1) + logp = Z @ basis + p = np.exp(logp) + return p / np.sum(p, axis=-1, keepdims=True) + + @lru_cache(maxsize=None) + def get_V(self, k): + helmert = np.zeros((k, k)) + for i in range(1, k): + helmert[i, :i] = 1 + helmert[i, i] = -i + helmert[i] = helmert[i] / np.sqrt(i * (i + 1)) + return helmert[1:, :] + + +def normalized_entropy(p): + """ + Computes the normalized Shannon entropy of a prevalence vector. + + :param p: array-like prevalence vector summing to 1 + :return: float in [0,1] + """ + p = np.asarray(p) + entropy = scipy.stats.entropy(p) + max_entropy = np.log(len(p)) + return np.clip(entropy / max_entropy, 0, 1) + + +def antagonistic_prevalence(p, strength=1): + """ + Reflects a prevalence vector in ILR space and maps it back to the simplex. + + :param p: array-like prevalence vector + :param strength: reflection strength in ILR space + :return: prevalence vector in the simplex + """ + ilr = ILRtransformation() + z = ilr(p) + z_ant = -strength * z + return ilr.inverse(z_ant) + + +def in_simplex(x, atol=1e-8): + """ + Checks whether points lie in the probability simplex. + + :param x: array-like of shape `(n_classes,)` or `(n_points, n_classes)` + :param atol: numerical tolerance for the unit-sum check + :return: boolean or boolean array + """ + x = np.asarray(x) + non_negative = np.all(x >= 0, axis=-1) + sum_to_one = np.isclose(x.sum(axis=-1), 1.0, atol=atol) + return non_negative & sum_to_one diff --git a/quapy/method/__init__.py b/quapy/method/__init__.py index 4c2ec1c..ff6a769 100644 --- a/quapy/method/__init__.py +++ b/quapy/method/__init__.py @@ -3,6 +3,7 @@ from sklearn.exceptions import ConvergenceWarning warnings.simplefilter("ignore", ConvergenceWarning) from . import confidence +from . import _bayesian from . import base from . import aggregative from . import non_aggregative @@ -14,6 +15,7 @@ AGGREGATIVE_METHODS = { aggregative.ACC, aggregative.PCC, aggregative.PACC, + aggregative.RLLS, aggregative.EMQ, aggregative.HDy, aggregative.DyS, @@ -24,11 +26,15 @@ AGGREGATIVE_METHODS = { aggregative.MS, aggregative.MS2, aggregative.DMy, + aggregative.EDy, aggregative.KDEyML, aggregative.KDEyCS, aggregative.KDEyHD, # aggregative.OneVsAllAggregative, confidence.BayesianCC, + _bayesian.BayesianKDEy, + _bayesian.BayesianMAPLS, + confidence.PQ, } BINARY_METHODS = { @@ -40,6 +46,7 @@ BINARY_METHODS = { aggregative.MAX, aggregative.MS, aggregative.MS2, + confidence.PQ, } MULTICLASS_METHODS = { @@ -47,16 +54,22 @@ MULTICLASS_METHODS = { aggregative.ACC, aggregative.PCC, aggregative.PACC, + aggregative.RLLS, aggregative.EMQ, + aggregative.EDy, aggregative.KDEyML, aggregative.KDEyCS, aggregative.KDEyHD, - confidence.BayesianCC + confidence.BayesianCC, + _bayesian.BayesianKDEy, + _bayesian.BayesianMAPLS, + non_aggregative.EDx, } NON_AGGREGATIVE_METHODS = { non_aggregative.MaximumLikelihoodPrevalenceEstimation, - non_aggregative.DMx + non_aggregative.DMx, + non_aggregative.EDx } META_METHODS = { @@ -68,5 +81,3 @@ QUANTIFICATION_METHODS = AGGREGATIVE_METHODS | NON_AGGREGATIVE_METHODS | META_ME - - diff --git a/quapy/method/_bayesian.py b/quapy/method/_bayesian.py index c783f10..d5d582f 100644 --- a/quapy/method/_bayesian.py +++ b/quapy/method/_bayesian.py @@ -1,20 +1,61 @@ """ -Utility functions for `Bayesian quantification `_ methods. +Utilities and methods for Bayesian quantification. """ +import contextlib +import copy +import importlib.resources +import logging +import os +import sys +import warnings +from collections.abc import Iterable +from numbers import Number, Real + import numpy as np +from joblib import Parallel, delayed +from sklearn.base import BaseEstimator +from tqdm import tqdm + +import quapy as qp +import quapy.functional as F +from quapy.data import LabelledCollection +from quapy.method._kdey import KDEBase +from quapy.method.aggregative import AggregativeSoftQuantifier +from quapy.method.confidence import ConfidenceRegionABC, WithConfidenceABC +from quapy.protocol import AbstractProtocol + +# stan's plugin discovery (stan.plugins.get_plugins) calls pkg_resources.iter_entry_points() +# on every model build, each of which re-emits setuptools' pkg_resources deprecation notice; +# this is upstream pystan noise, not actionable in quapy, so it is silenced here. +warnings.filterwarnings( + "ignore", + message=r".*pkg_resources is deprecated.*", + category=UserWarning, +) try: import jax import jax.numpy as jnp + import jax.random as jrandom + from jax.scipy.special import logsumexp as jax_logsumexp import numpyro import numpyro.distributions as dist + from numpyro.infer import MCMC, NUTS + import stan + import stan.common DEPENDENCIES_INSTALLED = True -except ImportError: +except ImportError as e: + logging.getLogger(__name__).warning(f'Bayesian dependencies failed to import: {e!r}') jax = None jnp = None + jrandom = None + jax_logsumexp = None numpyro = None dist = None + MCMC = None + NUTS = None + stan = None DEPENDENCIES_INSTALLED = False @@ -24,28 +65,124 @@ P_TEST_C: str = "P_test(C)" P_C_COND_Y: str = "P(C|Y)" -def model(n_c_unlabeled: np.ndarray, n_y_and_c_labeled: np.ndarray) -> None: - """ - Defines a probabilistic model in `NumPyro `_. +def _require_bayesian_dependencies(): + if not DEPENDENCIES_INSTALLED: + raise ImportError( + "Auxiliary dependencies are required. " + "Run `$ pip install quapy[bayes]` to install them." + ) - :param n_c_unlabeled: a `np.ndarray` of shape `(n_predicted_classes,)` - with entry `c` being the number of instances predicted as class `c`. - :param n_y_and_c_labeled: a `np.ndarray` of shape `(n_classes, n_predicted_classes)` - with entry `(y, c)` being the number of instances labeled as class `y` and predicted as class `c`. + +def _resolve_dirichlet_prior(prior, n_classes, *, allow_mapls_priors=False, n_test=None, map_prev=None, map_lambda=None): + if isinstance(prior, str): + if prior == 'uniform': + return np.ones(n_classes, dtype=float) + if allow_mapls_priors and prior in {'map', 'map2'}: + if n_test is None or map_prev is None: + raise ValueError('MAPLS priors require n_test and map_prev') + if prior == 'map': + lam = map_lambda + else: + lam = get_lambda( + test_probs=map_prev["test_probs"], + pz=map_prev["train_prev"], + q_prior=map_prev["map_estimate"], + dvg=kl_div, + ) + alpha_0 = alpha0_from_lambda(lam, n_test=n_test, n_classes=n_classes) + return np.full(n_classes, alpha_0, dtype=float) + raise ValueError(f"unknown prior specification {prior!r}") + if isinstance(prior, Number): + return np.full(n_classes, float(prior), dtype=float) + + alpha = np.asarray(prior, dtype=float) + if alpha.ndim != 1 or len(alpha) != n_classes: + raise ValueError(f'wrong shape for prior; expected {n_classes} values, found shape {alpha.shape}') + return alpha + + +def _validate_temperature(temperature): + if not isinstance(temperature, Real) or temperature <= 0: + raise ValueError(f'expected a positive real value for temperature; found {temperature!r}') + return float(temperature) + + +def model_bayesianCC( + n_c_unlabeled: np.ndarray, + n_y_and_c_labeled: np.ndarray, + temperature: float, + alpha: np.ndarray, +) -> None: + """ + NumPyro model for BayesianCC. """ n_y_labeled = n_y_and_c_labeled.sum(axis=1) - K = len(n_c_unlabeled) - L = len(n_y_labeled) + n_pred_classes = len(n_c_unlabeled) + n_classes = len(n_y_labeled) - pi_ = numpyro.sample(P_TEST_Y, dist.Dirichlet(jnp.ones(L))) - p_c_cond_y = numpyro.sample(P_C_COND_Y, dist.Dirichlet(jnp.ones(K).repeat(L).reshape(L, K))) + pi_ = numpyro.sample(P_TEST_Y, dist.Dirichlet(jnp.asarray(alpha, dtype=jnp.float32))) + p_c_cond_y = numpyro.sample( + P_C_COND_Y, + dist.Dirichlet(jnp.ones(n_pred_classes).repeat(n_classes).reshape(n_classes, n_pred_classes)), + ) - with numpyro.plate('plate', L): - numpyro.sample('F_yc', dist.Multinomial(n_y_labeled, p_c_cond_y), obs=n_y_and_c_labeled) + if temperature == 1.0: + with numpyro.plate('plate', n_classes): + numpyro.sample('F_yc', dist.Multinomial(n_y_labeled, p_c_cond_y), obs=n_y_and_c_labeled) + + p_c = numpyro.deterministic(P_TEST_C, jnp.einsum("yc,y->c", p_c_cond_y, pi_)) + numpyro.sample('N_c', dist.Multinomial(jnp.sum(n_c_unlabeled), p_c), obs=n_c_unlabeled) + return + + with numpyro.plate('plate_y', n_classes): + logp_f = dist.Multinomial(n_y_labeled, p_c_cond_y).log_prob(n_y_and_c_labeled) + + numpyro.factor('F_yc_loglik', jnp.sum(logp_f) / temperature) p_c = numpyro.deterministic(P_TEST_C, jnp.einsum("yc,y->c", p_c_cond_y, pi_)) - numpyro.sample('N_c', dist.Multinomial(jnp.sum(n_c_unlabeled), p_c), obs=n_c_unlabeled) + logp_n = dist.Multinomial(jnp.sum(n_c_unlabeled), p_c).log_prob(n_c_unlabeled) + numpyro.factor('N_c_loglik', logp_n / temperature) + + +def model(n_c_unlabeled: np.ndarray, n_y_and_c_labeled: np.ndarray) -> None: + """ + Backward-compatible BayesianCC model with a uniform prior. + """ + alpha = np.ones(n_y_and_c_labeled.shape[0], dtype=float) + return model_bayesianCC(n_c_unlabeled, n_y_and_c_labeled, temperature=1.0, alpha=alpha) + + +def sample_posterior_bayesianCC( + n_c_unlabeled: np.ndarray, + n_y_and_c_labeled: np.ndarray, + num_warmup: int, + num_samples: int, + alpha: np.ndarray, + temperature: float = 1.0, + seed: int = 0, +) -> dict: + """ + Samples from the BayesianCC posterior using NumPyro. + """ + _require_bayesian_dependencies() + temperature = _validate_temperature(temperature) + + mcmc = numpyro.infer.MCMC( + numpyro.infer.NUTS(model_bayesianCC), + num_warmup=num_warmup, + num_samples=num_samples, + progress_bar=False, + ) + rng_key = jax.random.PRNGKey(seed) + mcmc.run( + rng_key, + n_c_unlabeled=n_c_unlabeled, + n_y_and_c_labeled=n_y_and_c_labeled, + temperature=temperature, + alpha=alpha, + ) + return mcmc.get_samples() def sample_posterior( @@ -56,24 +193,691 @@ def sample_posterior( seed: int = 0, ) -> dict: """ - Samples from the Bayesian quantification model in NumPyro using the - `NUTS `_ sampler. - - :param n_c_unlabeled: a `np.ndarray` of shape `(n_predicted_classes,)` - with entry `c` being the number of instances predicted as class `c`. - :param n_y_and_c_labeled: a `np.ndarray` of shape `(n_classes, n_predicted_classes)` - with entry `(y, c)` being the number of instances labeled as class `y` and predicted as class `c`. - :param num_warmup: the number of warmup steps. - :param num_samples: the number of samples to draw. - :seed: the random seed. - :return: a `dict` with the samples. The keys are the names of the latent variables. + Backward-compatible wrapper around BayesianCC sampling. """ - mcmc = numpyro.infer.MCMC( - numpyro.infer.NUTS(model), + alpha = np.ones(n_y_and_c_labeled.shape[0], dtype=float) + return sample_posterior_bayesianCC( + n_c_unlabeled=n_c_unlabeled, + n_y_and_c_labeled=n_y_and_c_labeled, num_warmup=num_warmup, num_samples=num_samples, - progress_bar=False + alpha=alpha, + temperature=1.0, + seed=seed, ) - rng_key = jax.random.PRNGKey(seed) - mcmc.run(rng_key, n_c_unlabeled=n_c_unlabeled, n_y_and_c_labeled=n_y_and_c_labeled) - return mcmc.get_samples() + + +def load_stan_file(): + return importlib.resources.files('quapy.method').joinpath('stan/pq.stan').read_text(encoding='utf-8') + + +@contextlib.contextmanager +def _suppress_stan_logging(): + with open(os.devnull, "w") as devnull: + old_stderr = sys.stderr + sys.stderr = devnull + try: + yield + finally: + sys.stderr = old_stderr + + +def pq_stan(stan_code, n_bins, pos_hist, neg_hist, test_hist, number_of_samples, num_warmup, stan_seed): + """ + Samples posterior prevalences for PQ from a Stan model. + """ + _require_bayesian_dependencies() + logging.getLogger("stan.common").setLevel(logging.ERROR) + + stan_data = { + 'n_bucket': n_bins, + 'train_neg': neg_hist.tolist(), + 'train_pos': pos_hist.tolist(), + 'test': test_hist.tolist(), + 'posterior': 1, + } + + with _suppress_stan_logging(): + stan_model = stan.build(stan_code, data=stan_data, random_seed=stan_seed) + fit = stan_model.sample(num_chains=1, num_samples=number_of_samples, num_warmup=num_warmup) + + return fit['prev'] + + +class BayesianKDEy(AggregativeSoftQuantifier, KDEBase, WithConfidenceABC): + """ + Bayesian version of KDEy. + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case + the classifier is taken to be the one indicated in + `qp.environ['DEFAULT_CLS']` + :param fit_classifier: whether to train the classifier, or consider it + already fit + :param val_split: specifies the data used for generating classifier + predictions. This specification can be made as float in (0, 1) + indicating the proportion of stratified held-out validation set to be + extracted from the training set; or as an integer (default 5), + indicating that the predictions are to be generated in a `k`-fold + cross-validation manner (with this integer indicating the value for + `k`); or as a tuple `(X,y)` defining the specific set of data to use + for validation. Set to None when the method does not require any + validation data, in order to avoid that some portion of the training + data be wasted. + :param kernel: kernel function for KDE. Available kernels include + {'gaussian', 'aitchison', 'ilr'} (default 'gaussian') + :param bandwidth: bandwidth for the kernel (default 0.1) + :param shrinkage: regularization strength for Aitchison/ILR kernels + (default 0.0) + :param num_warmup: number of warmup iterations for the MCMC sampler + (default 500) + :param num_samples: number of samples to draw from the posterior + (default 1000) + :param mcmc_seed: random seed for the MCMC sampler (default 0) + :param confidence_level: float in [0,1] to construct a confidence region + around the point estimate (default 0.95) + :param region: string, set to `intervals` for constructing confidence + intervals (default), or to `ellipse` for constructing an ellipse in + the probability simplex, or to `ellipse-clr` for constructing an + ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni + correction when `region='intervals'`. This parameter has no effect + for ellipse-based regions. + :param temperature: temperature (>0) for posterior calibration + (default 1.) + :param prior: an array-like with the alpha parameters of a Dirichlet + prior, a scalar real value to be broadcast to all classes, or the + string 'uniform' for a uniform, uninformative prior (default) + :param verbose: bool, whether to display the progress bar + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier=True, + val_split: int = 5, + kernel='gaussian', + bandwidth=0.1, + shrinkage=0.0, + num_warmup: int = 500, + num_samples: int = 1_000, + mcmc_seed: int = 0, + confidence_level: float = 0.95, + region: str = 'intervals', + bonferroni: bool = False, + temperature: float = 1.0, + prior='uniform', + verbose: bool = False, + ): + _require_bayesian_dependencies() + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + + self.kernel = KDEBase._check_kernel(kernel) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, self.kernel) + assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)' + assert self.kernel != 'gaussian' or shrinkage == 0, \ + 'shrinkage is only supported for Aitchison/ILR kernels' + + super().__init__(classifier, fit_classifier, val_split) + self.shrinkage = float(shrinkage) + self.num_warmup = num_warmup + self.num_samples = num_samples + self.mcmc_seed = mcmc_seed + self.confidence_level = confidence_level + self.region = region + self.bonferroni = bonferroni + self.temperature = _validate_temperature(temperature) + self.prior = prior + self.verbose = verbose + self.prevalence_samples = None + + def aggregation_fit(self, classif_predictions, labels): + self.mix_densities = self.get_mixture_components( + classif_predictions, + labels, + self.classes_, + self.bandwidth, + self.kernel, + ) + return self + + def sample_from_posterior(self, classif_predictions): + test_log_densities = np.asarray( + [self.pdf(kde_i, classif_predictions, self.kernel, log_densities=True) for kde_i in self.mix_densities] + ) + alpha = _resolve_dirichlet_prior(self.prior, len(self.mix_densities)) + + mcmc = MCMC( + NUTS(self._numpyro_model), + num_warmup=self.num_warmup, + num_samples=self.num_samples, + num_chains=1, + progress_bar=self.verbose, + ) + mcmc.run(jrandom.PRNGKey(self.mcmc_seed), test_log_densities=test_log_densities, alpha=alpha) + self.prevalence_samples = np.asarray(mcmc.get_samples()["prev"]) + return self.prevalence_samples + + def aggregate(self, classif_predictions: np.ndarray): + return self.sample_from_posterior(classif_predictions).mean(axis=0) + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + confidence_level = self.confidence_level if confidence_level is None else confidence_level + classif_predictions = self.classify(instances) + point_estimate = self.aggregate(classif_predictions) + region = WithConfidenceABC.construct_region( + self.prevalence_samples, + confidence_level=confidence_level, + method=self.region, + bonferroni=self.bonferroni, + ) + return point_estimate, region + + def _numpyro_model(self, test_log_densities, alpha): + prev = numpyro.sample("prev", dist.Dirichlet(jnp.asarray(alpha))) + log_likelihood = jnp.sum(jax_logsumexp(jnp.log(prev)[:, None] + test_log_densities, axis=0)) + numpyro.factor("loglik", (1.0 / self.temperature) * log_likelihood) + + +class _JaxILRTransformation(F.CompositionalTransformation): + """ + JAX-backed ILR transform used inside Bayesian MAPLS. + """ + + def __call__(self, X): + X = jnp.asarray(X) + X = qp.error.smooth(np.asarray(X), self.EPSILON) + X = jnp.asarray(X) + basis = jnp.asarray(self.get_V(X.shape[-1])) + return jnp.log(X) @ basis.T + + def inverse(self, Z): + Z = jnp.asarray(Z) + basis = jnp.asarray(self.get_V(Z.shape[-1] + 1)) + logp = Z @ basis + p = jnp.exp(logp) + return p / jnp.sum(p, axis=-1, keepdims=True) + + def get_V(self, k): + return F.ILRtransformation().get_V(k) + + +class BayesianMAPLS(AggregativeSoftQuantifier, WithConfidenceABC): + """ + Bayesian variant of the MLLS/EMQ method proposed by + Ye, Changkun, et al. "Label shift estimation for class-imbalance problem: + A bayesian approach." Proceedings of the IEEE/CVF Winter Conference on + Applications of Computer Vision. 2024. + + Code adapted from: + https://github.com/ChangkunYe/MAPLS/blob/main/label_shift/mapls.py + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case + the classifier is taken to be the one indicated in + `qp.environ['DEFAULT_CLS']` + :param fit_classifier: whether to train the classifier, or consider it + already fit + :param val_split: specifies the data used for generating classifier + predictions. This specification can be made as float in (0, 1) + indicating the proportion of stratified held-out validation set to be + extracted from the training set; or as an integer (default 5), + indicating that the predictions are to be generated in a `k`-fold + cross-validation manner (with this integer indicating the value for + `k`); or as a tuple `(X,y)` defining the specific set of data to use + for validation. Set to None when the method does not require any + validation data, in order to avoid that some portion of the training + data be wasted. + :param exact_train_prev: set to True (default) for using the true training + prevalence as the initial observation; set to False for computing the + training prevalence as an estimate of it, i.e., as the expected value + of the posterior probabilities of the training instances. + :param num_warmup: number of warmup iterations for the MCMC sampler + (default 500) + :param num_samples: number of samples to draw from the posterior + (default 1000) + :param mcmc_seed: random seed for the MCMC sampler (default 0) + :param confidence_level: float in [0,1] to construct a confidence region + around the point estimate (default 0.95) + :param region: string, set to `intervals` for constructing confidence + intervals (default), or to `ellipse` for constructing an ellipse in + the probability simplex, or to `ellipse-clr` for constructing an + ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni + correction when `region='intervals'`. This parameter has no effect + for ellipse-based regions. + :param temperature: temperature (>0) for posterior calibration + (default 1.) + :param prior: an array-like with the alpha parameters of a Dirichlet + prior, a scalar real value to be broadcast to all classes, or one of + {'uniform', 'map', 'map2'} (default 'uniform') + :param mapls_chain_init: whether to initialize the Markov chain with a + preliminary EM point estimate (default True) + :param verbose: bool, whether to display the progress bar + (default False) + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier=True, + val_split: int = 5, + exact_train_prev=True, + num_warmup: int = 500, + num_samples: int = 1_000, + mcmc_seed: int = 0, + confidence_level: float = 0.95, + region: str = 'intervals', + bonferroni: bool = False, + temperature: float = 1.0, + prior='uniform', + mapls_chain_init=True, + verbose=False, + ): + _require_bayesian_dependencies() + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + if not ( + (isinstance(prior, str) and prior in {'uniform', 'map', 'map2'}) + or isinstance(prior, Number) + or (isinstance(prior, Iterable) and all(isinstance(v, Number) for v in prior)) + ): + raise ValueError( + f'wrong type for {prior=}; expected one of {{"uniform", "map", "map2"}}, ' + 'a real scalar, or an array-like of real values' + ) + + super().__init__(classifier, fit_classifier, val_split) + self.exact_train_prev = exact_train_prev + self.num_warmup = num_warmup + self.num_samples = num_samples + self.mcmc_seed = mcmc_seed + self.confidence_level = confidence_level + self.region = region + self.bonferroni = bonferroni + self.temperature = _validate_temperature(temperature) + self.prior = prior + self.mapls_chain_init = mapls_chain_init + self.verbose = verbose + self.prevalence_samples = None + + def aggregation_fit(self, classif_predictions, labels): + self.train_post = classif_predictions + if self.exact_train_prev: + self.train_prevalence = F.prevalence_from_labels(labels, classes=self.classes_) + else: + self.train_prevalence = F.prevalence_from_probabilities(classif_predictions) + self.ilr = _JaxILRTransformation() + return self + + def sample_from_posterior(self, classif_predictions): + n_test, n_classes = classif_predictions.shape + map_estimate, lam = mapls( + self.train_post, + test_probs=classif_predictions, + pz=self.train_prevalence, + return_lambda=True, + ) + + z0 = self.ilr(map_estimate) + if isinstance(self.prior, str) and self.prior in {'map', 'map2'}: + prior_context = { + "test_probs": classif_predictions, + "train_prev": self.train_prevalence, + "map_estimate": map_estimate, + } + alpha = _resolve_dirichlet_prior( + self.prior, + n_classes, + allow_mapls_priors=True, + n_test=n_test, + map_prev=prior_context, + map_lambda=lam, + ) + else: + alpha = _resolve_dirichlet_prior(self.prior, n_classes) + + mcmc = MCMC( + NUTS(self._numpyro_model), + num_warmup=self.num_warmup, + num_samples=self.num_samples, + num_chains=1, + progress_bar=self.verbose, + ) + mcmc.run( + jrandom.PRNGKey(self.mcmc_seed), + test_posteriors=classif_predictions, + alpha=alpha, + init_params={"z": z0} if self.mapls_chain_init else None, + ) + + samples = mcmc.get_samples()["z"] + self.prevalence_samples = np.asarray(self.ilr.inverse(samples)) + return self.prevalence_samples + + def aggregate(self, classif_predictions: np.ndarray): + return self.sample_from_posterior(classif_predictions).mean(axis=0) + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + confidence_level = self.confidence_level if confidence_level is None else confidence_level + classif_predictions = self.classify(instances) + point_estimate = self.aggregate(classif_predictions) + region = WithConfidenceABC.construct_region( + self.prevalence_samples, + confidence_level=confidence_level, + method=self.region, + bonferroni=self.bonferroni, + ) + return point_estimate, region + + def _log_likelihood(self, test_classif, test_prev, train_prev): + log_w = jnp.log(test_prev) - jnp.log(train_prev) + return jnp.sum(jax_logsumexp(jnp.log(test_classif) + log_w, axis=-1)) + + def _numpyro_model(self, test_posteriors, alpha): + test_posteriors = jnp.asarray(test_posteriors) + n_classes = test_posteriors.shape[1] + + z = numpyro.sample("z", dist.Normal(jnp.zeros(n_classes - 1), 1.0)) + prev = self.ilr.inverse(z) + train_prev = jnp.asarray(self.train_prevalence) + alpha = jnp.asarray(alpha) + + numpyro.factor("dirichlet_prior", dist.Dirichlet(alpha).log_prob(prev)) + numpyro.factor( + "likelihood", + (1.0 / self.temperature) * self._log_likelihood(test_posteriors, test_prev=prev, train_prev=train_prev), + ) + + +def mapls( + train_probs: np.ndarray, + test_probs: np.ndarray, + pz: np.ndarray, + qy_mode: str = 'soft', + max_iter: int = 100, + init_mode: str = 'identical', + lam: float = None, + dvg_name='kl', + return_lambda=False, +): + cls_num = len(pz) + assert test_probs.shape[-1] == cls_num + if not isinstance(max_iter, int) or max_iter < 0: + raise ValueError(f'expected a non-negative integer for max_iter; found {max_iter!r}') + + if dvg_name == 'kl': + dvg = kl_div + elif dvg_name == 'js': + dvg = js_div + else: + raise ValueError(f'Unsupported distribution distance measure {dvg_name!r}; expected "kl" or "js"') + + q_prior = np.ones(cls_num) / cls_num + if lam is None: + lam = get_lambda(test_probs, pz, q_prior, dvg=dvg, max_iter=max_iter) + + qz = mapls_em( + test_probs, + pz, + lam, + q_prior, + cls_num, + init_mode=init_mode, + max_iter=max_iter, + qy_mode=qy_mode, + ) + return (qz, lam) if return_lambda else qz + + +def mapls_em(probs, pz, lam, q_prior, cls_num, init_mode='identical', max_iter=100, qy_mode='soft'): + pz = np.asarray(pz, dtype=float) + pz = pz / np.sum(pz) + if init_mode == 'uniform': + qz = np.ones(cls_num) / cls_num + elif init_mode == 'identical': + qz = pz.copy() + else: + raise ValueError('init_mode should be either "uniform" or "identical"') + + w = qz / pz + for _ in range(max_iter): + mapls_probs = normalized(probs * w, axis=-1, order=1) + if qy_mode == 'hard': + pred = np.argmax(mapls_probs, axis=-1) + qz_new = np.bincount(pred.reshape(-1), minlength=cls_num) + elif qy_mode == 'soft': + qz_new = np.mean(mapls_probs, axis=0) + else: + raise ValueError('qy_mode should be either "soft" or "hard"') + + qz = lam * qz_new + (1 - lam) * q_prior + qz /= qz.sum() + w = qz / pz + + return qz + + +def get_lambda(test_probs, pz, q_prior, dvg, max_iter=50): + n_classes = len(pz) + qz_pred = mapls_em(test_probs, pz, 1, 0, n_classes, max_iter=max_iter) + + tu_div = dvg(qz_pred, q_prior) + ts_div = dvg(qz_pred, pz) + su_div = dvg(pz, q_prior) + + su_conf = 1 - lambda_forward(su_div, lambda_inverse(dpq=0.5, lam=0.2)) + tu_conf = lambda_forward(tu_div, lambda_inverse(dpq=0.5, lam=su_conf)) + ts_conf = lambda_forward(ts_div, lambda_inverse(dpq=0.5, lam=su_conf)) + + confs = np.array([tu_conf, 1 - ts_conf]) + weights = np.array([0.9, 0.1]) + return np.sum(weights * confs) + + +def lambda_inverse(dpq, lam): + return (1 / (1 - lam) - 1) / dpq + + +def lambda_forward(dpq, gamma): + return gamma * dpq / (1 + gamma * dpq) + + +def get_lamda(test_probs, pz, q_prior, dvg, max_iter=50): + return get_lambda(test_probs, pz, q_prior, dvg, max_iter=max_iter) + + +def lam_inv(dpq, lam): + return lambda_inverse(dpq, lam) + + +def lam_forward(dpq, gamma): + return lambda_forward(dpq, gamma) + + +def kl_div(p, q, eps=1e-12): + p = np.asarray(p, dtype=float) + q = np.asarray(q, dtype=float) + + mask = p > 0 + return np.sum(p[mask] * np.log(p[mask] / (q[mask] + eps))) + + +def js_div(p, q): + assert (np.abs(np.sum(p) - 1) < 1e-6) and (np.abs(np.sum(q) - 1) < 1e-6) + m = (p + q) / 2 + return kl_div(p, m) / 2 + kl_div(q, m) / 2 + + +def normalized(a, axis=-1, order=2): + l2 = np.atleast_1d(np.linalg.norm(a, order, axis)) + l2[l2 == 0] = 1 + return a / np.expand_dims(l2, axis) + + +def alpha0_from_lambda(lam, n_test, n_classes): + return 1 + n_test * (1 - lam) / (lam * n_classes) + + +def alpha0_from_lamda(lam, n_test, n_classes): + return alpha0_from_lambda(lam, n_test, n_classes) + + +def calibrate_temperature( + method: WithConfidenceABC, + train: LabelledCollection, + val_prot: AbstractProtocol, + temp_grid=(0.5, 1.0, 1.5, 2.0, 5.0, 10.0, 100.0), + nominal_coverage: float = 0.95, + amplitude_threshold=1.0, + criterion: str = 'winkler', + n_jobs: int = 1, + verbose: bool = True, +): + """ + Calibrates the temperature parameter of a Bayesian quantifier with + confidence regions by selecting the value that yields the best validation + trade-off between nominal coverage and region sharpness. + + The method is first fitted on ``train``. For each candidate temperature, + the fitted quantifier is deep-copied, its ``temperature`` attribute is + replaced, and it is evaluated on the samples generated by ``val_prot``. + Candidate temperatures whose average region amplitude exceeds + ``amplitude_threshold`` are discarded. + + When ``criterion='winkler'``, the surviving candidate with minimum mean + Winkler score is selected. When ``criterion='auto'``, the selected + temperature is the one whose empirical coverage is closest to + ``nominal_coverage``. + + :param method: a quantifier implementing :class:`WithConfidenceABC` and + exposing a writable ``temperature`` attribute + :param train: training set used to fit the quantifier + :param val_prot: validation protocol yielding pairs ``(sample, true_prev)`` + :param temp_grid: candidate temperatures to evaluate + :param nominal_coverage: target confidence level used by the Winkler score + and coverage selection + :param amplitude_threshold: maximum allowed average simplex proportion of + the region. It can also be set to ``'auto'`` to use a heuristic based + on the number of classes + :param criterion: either ``'winkler'`` (default) or ``'auto'`` + :param n_jobs: number of parallel jobs across candidate temperatures + :param verbose: whether to display progress information + :return: the selected temperature value + """ + if not hasattr(method, 'temperature'): + raise ValueError(f'{method.__class__.__name__} does not expose a temperature attribute') + if not isinstance(method, WithConfidenceABC): + raise TypeError(f'{method.__class__.__name__} is not an instance of WithConfidenceABC') + if not 0 < nominal_coverage < 1: + raise ValueError(f'{nominal_coverage=} must be in the interval (0,1)') + if criterion not in {'auto', 'winkler'}: + raise ValueError(f'unknown {criterion=}; valid ones are "auto" or "winkler"') + if amplitude_threshold != 'auto': + if not isinstance(amplitude_threshold, Real) or amplitude_threshold > 1.0: + raise ValueError( + f'wrong value for {amplitude_threshold=}; it must either be "auto" or a real value <= 1.0' + ) + temperatures = sorted(_validate_temperature(temp) for temp in temp_grid) + + if amplitude_threshold == 'auto': + amplitude_threshold = 0.1 / np.log(train.n_classes + 1) + + if amplitude_threshold > 0.1: + print(f'warning: the {amplitude_threshold=} is too large; this may lead to uninformative regions') + + def _evaluate_temperature_job(job_id, temperature): + local_method = copy.deepcopy(method) + local_method.temperature = temperature + + coverage = 0 + amplitudes = [] + winklers = [] + errors = [] + + pbar = tqdm( + enumerate(val_prot()), + position=job_id, + total=val_prot.total(), + disable=not verbose, + ) + + for i, (sample, prev) in pbar: + point_estim, conf_region = local_method.predict_conf(sample) + + if prev in conf_region: + coverage += 1 + + amplitudes.append(conf_region.montecarlo_proportion(n_trials=50_000)) + if criterion == 'winkler': + winklers.append(conf_region.mean_winkler_score(true_prev=prev, alpha=1 - nominal_coverage)) + errors.append(qp.error.mae(prev, point_estim)) + + description = ( + f'job={job_id} T={temperature}: ' + f'MAE={np.mean(errors):.6f} ' + f'coverage={coverage / (i + 1) * 100:.2f}% ' + f'amplitude={np.mean(amplitudes) * 100:.4f}% ' + ) + if criterion == 'winkler': + description += f'winkler={np.mean(winklers):.4f}' + pbar.set_description(description) + + mean_coverage = coverage / val_prot.total() + mean_amplitude = float(np.mean(amplitudes)) + mean_winkler = float(np.mean(winklers)) if criterion == 'winkler' else None + mean_error = float(np.mean(errors)) + return temperature, mean_coverage, mean_amplitude, mean_winkler, mean_error + + method.fit(*train.Xy) + raw_results = Parallel(n_jobs=n_jobs, backend="loky")( + delayed(_evaluate_temperature_job)(job_id, temperature) + for job_id, temperature in tqdm(enumerate(temperatures), disable=not verbose) + ) + filtered_results = [ + (temperature, coverage, amplitude, winkler, error) + for temperature, coverage, amplitude, winkler, error in raw_results + if amplitude < amplitude_threshold + ] + + chosen_temperature = 1.0 + chosen_coverage = chosen_amplitude = chosen_winkler = chosen_error = None + + if filtered_results: + if criterion == 'winkler': + chosen_temperature, chosen_coverage, chosen_amplitude, chosen_winkler, chosen_error = min( + filtered_results, key=lambda item: item[3] + ) + else: + chosen_temperature, chosen_coverage, chosen_amplitude, chosen_winkler, chosen_error = min( + filtered_results, key=lambda item: abs(item[1] - nominal_coverage) + ) + + if verbose and chosen_coverage is not None: + message = ( + f'\nChosen_temperature={chosen_temperature:.2f} got ' + f'MAE={chosen_error:.6f} ' + f'coverage={chosen_coverage * 100:.2f}% ' + f'amplitude={chosen_amplitude * 100:.4f}% ' + ) + if criterion == 'winkler': + message += f'winkler={chosen_winkler:.4f}' + print(message) + + return chosen_temperature + + +def temp_calibration(*args, **kwargs): + """ + Backward-compatible alias for :func:`calibrate_temperature`. + """ + return calibrate_temperature(*args, **kwargs) diff --git a/quapy/method/_energy.py b/quapy/method/_energy.py new file mode 100644 index 0000000..e0ca8ed --- /dev/null +++ b/quapy/method/_energy.py @@ -0,0 +1,144 @@ +from typing import Callable, Union + +import numpy as np +from sklearn.metrics.pairwise import euclidean_distances, manhattan_distances + +import quapy as qp +import quapy.functional as F +from quapy.method._helper import _get_quadprog + + +class _EnergyDistanceCore: + """Shared numerical core for energy-distance quantifiers.""" + + def _check_ed_init_parameters(self): + self.distance = self._resolve_distance_function(self.distance) + + @staticmethod + def _resolve_distance_function(distance): + if isinstance(distance, str): + if distance == 'manhattan': + return manhattan_distances + if distance == 'euclidean': + return euclidean_distances + raise ValueError( + f"unknown distance {distance!r}; valid aliases are 'manhattan' and 'euclidean'" + ) + if not hasattr(distance, '__call__'): + raise ValueError('distance must be a valid string alias or a callable function') + return distance + + def _is_pd(self, m): + """Check whether a symmetric matrix is positive definite.""" + return self._dpofa(m)[0] == 0 + + def _dpofa(self, m): + """Factor a symmetric positive definite matrix.""" + r = np.array(m, copy=True) + n = len(r) + for k in range(n): + s = 0.0 + if k >= 1: + for i in range(k): + t = r[i, k] + if i > 0: + t = t - np.sum(r[0:i, i] * r[0:i, k]) + t = t / r[i, i] + r[i, k] = t + s = s + t * t + s = r[k, k] - s + if s <= 0.0: + return k + 1, r + r[k, k] = np.sqrt(s) + return 0, r + + def _nearest_pd(self, A): + """Project a matrix onto the cone of positive-definite matrices.""" + B = (A + A.T) / 2 + _, s, V = np.linalg.svd(B) + H = V.T @ np.diag(s) @ V + A2 = (B + H) / 2 + A3 = (A2 + A2.T) / 2 + + if self._is_pd(A3): + return A3 + + spacing = np.spacing(np.linalg.norm(A)) + identity_matrix = np.eye(A.shape[0]) + k = 1 + while not self._is_pd(A3): + mineig = np.min(np.real(np.linalg.eigvals(A3))) + A3 += identity_matrix * (-mineig * k ** 2 + spacing) + k += 1 + + return A3 + + def _compute_ed_param_train(self, distance_func, train_distrib, n_cls_i): + """Pre-compute the training-side terms of the ED optimization problem.""" + n_classes = len(train_distrib) + K = np.zeros((n_classes, n_classes), dtype=float) + for i in range(n_classes): + K[i, i] = distance_func(train_distrib[i], train_distrib[i]).sum() + for j in range(i + 1, n_classes): + K[i, j] = distance_func(train_distrib[i], train_distrib[j]).sum() + K[j, i] = K[i, j] + + K = K / np.dot(n_cls_i, n_cls_i.T) + + B = np.zeros((n_classes - 1, n_classes - 1), dtype=float) + for i in range(n_classes - 1): + B[i, i] = -K[i, i] - K[-1, -1] + 2 * K[i, -1] + for j in range(n_classes - 1): + if j == i: + continue + B[i, j] = -K[i, j] - K[-1, -1] + K[i, -1] + K[j, -1] + + G = 2 * B + if not self._is_pd(G): + G = self._nearest_pd(G) + + C = -np.vstack([np.ones((1, n_classes - 1)), -np.eye(n_classes - 1)]).T + b = -np.array([1] + [0] * (n_classes - 1), dtype=float) + + return K, G, C, b + + def _compute_ed_param_test(self, distance_func, train_distrib, test_distrib, K, n_cls_i): + """Compute the test-dependent linear term of the ED objective.""" + n_classes = len(train_distrib) + Kt = np.zeros(n_classes, dtype=float) + for i in range(n_classes): + Kt[i] = distance_func(train_distrib[i], test_distrib).sum() + + Kt = Kt / (n_cls_i.squeeze() * float(test_distrib.shape[0])) + return 2 * (-Kt[:-1] + K[:-1, -1] + Kt[-1] - K[-1, -1]) + + def _solve_ed(self, G, a, C, b): + """Solve the energy-distance quadratic program.""" + quadprog = _get_quadprog() + sol = quadprog.solve_qp(G=G, a=a, C=C, b=b) + prevalences = sol[0] + prevalences = np.append(prevalences, 1 - prevalences.sum()) + return F.normalize_prevalence(prevalences, method='clip') + + def _fit_energy_model(self, train_distrib): + self.train_distrib_ = tuple(train_distrib) + self.train_n_cls_i_ = np.asarray( + [[distrib.shape[0]] for distrib in self.train_distrib_], + dtype=float, + ) + self.K_, self.G_, self.C_, self.b_ = self._compute_ed_param_train( + self.distance, + self.train_distrib_, + self.train_n_cls_i_, + ) + return self + + def _predict_energy(self, test_distrib): + self.a_ = self._compute_ed_param_test( + self.distance, + self.train_distrib_, + test_distrib, + self.K_, + self.train_n_cls_i_, + ) + return self._solve_ed(G=self.G_, a=self.a_, C=self.C_, b=self.b_) diff --git a/quapy/method/_helper.py b/quapy/method/_helper.py new file mode 100644 index 0000000..48d0cb6 --- /dev/null +++ b/quapy/method/_helper.py @@ -0,0 +1,117 @@ +""" +Internal helper utilities shared by quantification methods. +""" + +import numpy as np +from sklearn.metrics import confusion_matrix +from sklearn.preprocessing import LabelEncoder + + +def _get_abstention_calibrators(): + try: + from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling + except ImportError as exc: + raise ImportError( + "Posterior calibration for EMQ requires the optional 'abstention' package." + ) from exc + return { + 'nbvs': NoBiasVectorScaling(), + 'bcts': TempScaling(bias_positions='all'), + 'ts': TempScaling(), + 'vs': VectorScaling(), + } + + +def _get_cvxpy(): + try: + import cvxpy as cp + except ImportError as exc: + raise ImportError( + "RLLS requires the optional 'cvxpy' package." + ) from exc + return cp + + +def _get_quadprog(): + try: + import quadprog + except ImportError as exc: + raise ImportError( + "EDy requires the optional 'quadprog' package." + ) from exc + return quadprog + + +def _labels_to_indices(labels, classes): + encoder = LabelEncoder().fit(classes) + return encoder.transform(labels) + + +def _rlls_check_mode(mode): + valid = {'soft', 'hard'} + if mode not in valid: + raise ValueError(f'unknown mode {mode!r}; valid ones are {valid}') + return mode + + +def _rlls_joint_distribution(posteriors, labels, classes, mode='soft'): + mode = _rlls_check_mode(mode) + posteriors = np.asarray(posteriors, dtype=float) + labels = np.asarray(labels) + n_samples, n_classes = posteriors.shape + assert n_classes == len(classes), 'wrong number of posterior columns' + + if mode == 'hard': + pred = np.argmax(posteriors, axis=1) + encoded_labels = _labels_to_indices(labels, classes) + joint = confusion_matrix(encoded_labels, pred, labels=np.arange(n_classes)).T.astype(float) + return joint / n_samples + + joint = np.zeros((n_classes, n_classes), dtype=float) + for class_index, class_ in enumerate(classes): + idx = labels == class_ + if idx.any(): + joint[:, class_index] = posteriors[idx].sum(axis=0) + return joint / n_samples + + +def _rlls_predicted_marginal(posteriors, mode='soft'): + mode = _rlls_check_mode(mode) + posteriors = np.asarray(posteriors, dtype=float) + if mode == 'soft': + return posteriors.mean(axis=0) + + pred = np.argmax(posteriors, axis=1) + counts = np.bincount(pred, minlength=posteriors.shape[1]).astype(float) + return counts / counts.sum() + + +def _rlls_compute_3deltaC(n_classes, n_train, delta): + return 3 * ( + 2 * np.log(2 * n_classes / delta) / (3 * n_train) + + np.sqrt(2 * np.log(2 * n_classes / delta) / n_train) + ) + + +def _rlls_compute_weights(C_zy, qz, pz, rho, clip=False): + cp = _get_cvxpy() + + n_classes = C_zy.shape[0] + theta = cp.Variable(n_classes) + b = qz - pz + objective = cp.Minimize(cp.pnorm(C_zy @ theta - b) + rho * cp.pnorm(theta)) + constraints = [-1 <= theta] + problem = cp.Problem(objective, constraints) + + try: + problem.solve(verbose=False, solver=cp.SCS) + except cp.error.SolverError: + problem.solve(verbose=False, solver=cp.SCS, use_indirect=True) + + if theta.value is None: + raise RuntimeError('RLLS optimization failed to produce a solution') + + w = 1 + np.asarray(theta.value, dtype=float) + if clip and np.any(w < 0): + w = np.clip(w, 0, None) + return w diff --git a/quapy/method/_kdey.py b/quapy/method/_kdey.py index 88613c2..8f12f24 100644 --- a/quapy/method/_kdey.py +++ b/quapy/method/_kdey.py @@ -1,11 +1,13 @@ import numpy as np +from numbers import Real from sklearn.base import BaseEstimator from sklearn.neighbors import KernelDensity import quapy as qp +from quapy.method._helper import _labels_to_indices from quapy.method.aggregative import AggregativeSoftQuantifier import quapy.functional as F - +from scipy.special import logsumexp from sklearn.metrics.pairwise import rbf_kernel @@ -15,44 +17,57 @@ class KDEBase: """ BANDWIDTH_METHOD = ['scott', 'silverman'] + KERNELS = ['gaussian', 'aitchison', 'ilr'] @classmethod - def _check_bandwidth(cls, bandwidth): + def _check_bandwidth(cls, bandwidth, kernel): """ Checks that the bandwidth parameter is correct :param bandwidth: either a string (see BANDWIDTH_METHOD) or a float :return: the bandwidth if the check is passed, or raises an exception for invalid values """ - assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, float), \ + assert bandwidth in KDEBase.BANDWIDTH_METHOD or isinstance(bandwidth, Real), \ f'invalid bandwidth, valid ones are {KDEBase.BANDWIDTH_METHOD} or float values' - if isinstance(bandwidth, float): - assert 0 < bandwidth < 1, \ - "the bandwith for KDEy should be in (0,1), since this method models the unit simplex" + if isinstance(bandwidth, Real): + bandwidth = float(bandwidth) return bandwidth - def get_kde_function(self, X, bandwidth): + @classmethod + def _check_kernel(cls, kernel): + assert kernel in KDEBase.KERNELS, f'unknown {kernel=}' + return kernel + + def get_kde_function(self, X, bandwidth, kernel): """ Wraps the KDE function from scikit-learn. :param X: data for which the density function is to be estimated :param bandwidth: the bandwidth of the kernel + :param kernel: the kernel family :return: a scikit-learn's KernelDensity object """ + X = self.transform_posteriors(X, kernel) + bandwidth = self.effective_bandwidth(bandwidth, kernel) return KernelDensity(bandwidth=bandwidth).fit(X) - def pdf(self, kde, X): + def pdf(self, kde, X, kernel, log_densities=False): """ Wraps the density evalution of scikit-learn's KDE. Scikit-learn returns log-scores (s), so this function returns :math:`e^{s}` :param kde: a previously fit KDE function :param X: the data for which the density is to be estimated + :param kernel: the kernel family :return: np.ndarray with the densities """ - return np.exp(kde.score_samples(X)) + X = self.transform_posteriors(X, kernel) + log_density = kde.score_samples(X) + if log_densities: + return log_density + return np.exp(log_density) - def get_mixture_components(self, X, y, classes, bandwidth): + def get_mixture_components(self, X, y, classes, bandwidth, kernel): """ Returns an array containing the mixture components, i.e., the KDE functions for each class. @@ -60,22 +75,57 @@ class KDEBase: :param y: the class labels :param n_classes: integer, the number of classes :param bandwidth: float, the bandwidth of the kernel + :param kernel: the kernel family :return: a list of KernelDensity objects, each fitted with the corresponding class-specific covariates """ class_cond_X = [] for cat in classes: selX = X[y==cat] if selX.size==0: - selX = [F.uniform_prevalence(len(classes))] + raise ValueError(f'empty class {cat}') class_cond_X.append(selX) - return [self.get_kde_function(X_cond_yi, bandwidth) for X_cond_yi in class_cond_X] + return [self.get_kde_function(X_cond_yi, bandwidth, kernel) for X_cond_yi in class_cond_X] + + def transform_posteriors(self, X, kernel): + if kernel in {'aitchison', 'ilr'}: + X = self.shrink_posteriors(X) + if kernel == 'aitchison': + return self.clr_transform(X) + if kernel == 'ilr': + return self.ilr_transform(X) + return X + + def shrink_posteriors(self, X): + shrinkage = getattr(self, 'shrinkage', 0.0) + if shrinkage <= 0: + return X + X = np.asarray(X) + n_classes = X.shape[-1] + uniform = np.full(n_classes, 1.0 / n_classes, dtype=X.dtype) + return (1.0 - shrinkage) * X + shrinkage * uniform + + def effective_bandwidth(self, bandwidth, kernel): + shrinkage = getattr(self, 'shrinkage', 0.0) + if shrinkage > 0 and kernel in {'aitchison', 'ilr'} and isinstance(bandwidth, Real): + return (1.0 - shrinkage) * float(bandwidth) + return bandwidth + + def clr_transform(self, X): + if not hasattr(self, 'clr'): + self.clr = F.CLRtransformation() + return self.clr(X) + + def ilr_transform(self, X): + if not hasattr(self, 'ilr'): + self.ilr = F.ILRtransformation() + return self.ilr(X) class KDEyML(AggregativeSoftQuantifier, KDEBase): """ Kernel Density Estimation model for quantification (KDEy) relying on the Kullback-Leibler divergence (KLD) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors show that minimizing the distribution mathing criterion for KLD is akin to performing maximum likelihood (ML). @@ -107,17 +157,31 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value for `k`); or as a tuple (X,y) defining the specific set of data to use for validation. :param bandwidth: float, the bandwidth of the Kernel + :param kernel: kernel of KDE, valid ones are in KDEBase.KERNELS + :param shrinkage: amount of shrinkage towards the uniform distribution to apply before + Aitchison/ILR transformations. Must be in ``[0,1)``. :param random_state: a seed to be set before fitting any base quantifier (default None) """ def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1, - random_state=None): + kernel='gaussian', shrinkage=0.0, random_state=None): super().__init__(classifier, fit_classifier, val_split) - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel) + self.kernel = self._check_kernel(kernel) + assert 0 <= shrinkage < 1, 'shrinkage must be in [0,1)' + assert self.kernel != 'gaussian' or shrinkage == 0, \ + 'shrinkage is only supported for Aitchison/ILR kernels' + self.shrinkage = float(shrinkage) self.random_state=random_state def aggregation_fit(self, classif_predictions, labels): - self.mix_densities = self.get_mixture_components(classif_predictions, labels, self.classes_, self.bandwidth) + self.mix_densities = self.get_mixture_components( + classif_predictions, + labels, + self.classes_, + self.bandwidth, + self.kernel, + ) return self def aggregate(self, posteriors: np.ndarray): @@ -129,14 +193,25 @@ class KDEyML(AggregativeSoftQuantifier, KDEBase): :return: a vector of class prevalence estimates """ with qp.util.temp_seed(self.random_state): - epsilon = 1e-10 + epsilon = 1e-12 n_classes = len(self.mix_densities) - test_densities = [self.pdf(kde_i, posteriors) for kde_i in self.mix_densities] + if (self.kernel != 'gaussian' and n_classes >= 20) or n_classes >= 30: + test_log_densities = [ + self.pdf(kde_i, posteriors, self.kernel, log_densities=True) + for kde_i in self.mix_densities + ] - def neg_loglikelihood(prev): - test_mixture_likelihood = sum(prev_i * dens_i for prev_i, dens_i in zip (prev, test_densities)) - test_loglikelihood = np.log(test_mixture_likelihood + epsilon) - return -np.sum(test_loglikelihood) + def neg_loglikelihood(prev): + prev = qp.error.smooth(prev, eps=epsilon) + test_loglikelihood = logsumexp(np.log(prev)[:, None] + test_log_densities, axis=0) + return -np.sum(test_loglikelihood) + else: + test_densities = [self.pdf(kde_i, posteriors, self.kernel) for kde_i in self.mix_densities] + + def neg_loglikelihood(prev): + test_mixture_likelihood = prev @ test_densities + test_loglikelihood = np.log(test_mixture_likelihood + epsilon) + return -np.sum(test_loglikelihood) return F.optim_minimize(neg_loglikelihood, n_classes) @@ -145,7 +220,7 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): """ Kernel Density Estimation model for quantification (KDEy) relying on the squared Hellinger Disntace (HD) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors proposed a Monte Carlo approach for minimizing the divergence. The distribution matching optimization problem comes down to solving: @@ -191,18 +266,22 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): super().__init__(classifier, fit_classifier, val_split) self.divergence = divergence - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian') self.random_state=random_state self.montecarlo_trials = montecarlo_trials def aggregation_fit(self, classif_predictions, labels): - self.mix_densities = self.get_mixture_components(classif_predictions, labels, self.classes_, self.bandwidth) + self.mix_densities = self.get_mixture_components( + classif_predictions, labels, self.classes_, self.bandwidth, 'gaussian' + ) N = self.montecarlo_trials rs = self.random_state n = len(self.classes_) self.reference_samples = np.vstack([kde_i.sample(N//n, random_state=rs) for kde_i in self.mix_densities]) - self.reference_classwise_densities = np.asarray([self.pdf(kde_j, self.reference_samples) for kde_j in self.mix_densities]) + self.reference_classwise_densities = np.asarray( + [self.pdf(kde_j, self.reference_samples, 'gaussian') for kde_j in self.mix_densities] + ) self.reference_density = np.mean(self.reference_classwise_densities, axis=0) # equiv. to (uniform @ self.reference_classwise_densities) return self @@ -212,8 +291,8 @@ class KDEyHD(AggregativeSoftQuantifier, KDEBase): # apply importance sampling (IS). In this version we compute D(p_alpha||q) with IS n_classes = len(self.mix_densities) - test_kde = self.get_kde_function(posteriors, self.bandwidth) - test_densities = self.pdf(test_kde, self.reference_samples) + test_kde = self.get_kde_function(posteriors, self.bandwidth, 'gaussian') + test_densities = self.pdf(test_kde, self.reference_samples, 'gaussian') def f_squared_hellinger(u): return (np.sqrt(u)-1)**2 @@ -243,7 +322,7 @@ class KDEyCS(AggregativeSoftQuantifier): """ Kernel Density Estimation model for quantification (KDEy) relying on the Cauchy-Schwarz divergence (CS) as the divergence measure to be minimized. This method was first proposed in the paper - `Kernel Density Estimation for Multiclass Quantification `_, in which + `Kernel Density Estimation for Multiclass Quantification `_ (`arXiv `_), in which the authors proposed a Monte Carlo approach for minimizing the divergence. The distribution matching optimization problem comes down to solving: @@ -278,7 +357,7 @@ class KDEyCS(AggregativeSoftQuantifier): def __init__(self, classifier: BaseEstimator=None, fit_classifier=True, val_split=5, bandwidth=0.1): super().__init__(classifier, fit_classifier, val_split) - self.bandwidth = KDEBase._check_bandwidth(bandwidth) + self.bandwidth = KDEBase._check_bandwidth(bandwidth, kernel='gaussian') def gram_matrix_mix_sum(self, X, Y=None): # this adapts the output of the rbf_kernel function (pairwise evaluations of Gaussian kernels k(x,y)) @@ -296,13 +375,11 @@ class KDEyCS(AggregativeSoftQuantifier): P, y = classif_predictions, labels n = len(self.classes_) - - assert all(sorted(np.unique(y)) == np.arange(n)), \ - 'label name gaps not allowed in current implementation' + y = _labels_to_indices(y, self.classes_) # counts_inv keeps track of the relative weight of each datapoint within its class # (i.e., the weight in its KDE model) - counts_inv = 1 / (F.counts_from_labels(y, classes=self.classes_)) + counts_inv = 1 / (F.counts_from_labels(y, classes=np.arange(n))) # tr_tr_sums corresponds to symbol \overline{B} in the paper tr_tr_sums = np.zeros(shape=(n,n), dtype=float) @@ -353,4 +430,3 @@ class KDEyCS(AggregativeSoftQuantifier): return partA + partB #+ partC return F.optim_minimize(divergence, n) - diff --git a/quapy/method/_neural.py b/quapy/method/_neural.py index 404090f..12d61b7 100644 --- a/quapy/method/_neural.py +++ b/quapy/method/_neural.py @@ -1,3 +1,4 @@ +import logging import os from pathlib import Path import random @@ -173,7 +174,7 @@ class QuaNetTrainer(BaseQuantifier): order_by=0 if data.binary else None, **self.quanet_params ).to(self.device) - print(self.quanet) + logging.getLogger(__name__).debug(self.quanet) self.optim = torch.optim.Adam(self.quanet.parameters(), lr=self.lr) early_stop = EarlyStop(self.patience, lower_is_better=True) @@ -188,8 +189,9 @@ class QuaNetTrainer(BaseQuantifier): if early_stop.IMPROVED: torch.save(self.quanet.state_dict(), checkpoint) elif early_stop.STOP: - print(f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' - f'for epoch {early_stop.best_epoch}') + logging.getLogger(__name__).info( + f'training ended by patience exhausted; loading best model parameters in {checkpoint} ' + f'for epoch {early_stop.best_epoch}') self.quanet.load_state_dict(torch.load(checkpoint)) break diff --git a/quapy/method/_threshold_optim.py b/quapy/method/_threshold_optim.py index 628f01a..9d7624a 100644 --- a/quapy/method/_threshold_optim.py +++ b/quapy/method/_threshold_optim.py @@ -110,10 +110,10 @@ class ThresholdOptimization(BinaryAggregativeQuantifier): TN = np.logical_and(y == y_, y == self.neg_label).sum() return TP, FP, FN, TN - def _compute_tpr(self, TP, FP): - if TP + FP == 0: + def _compute_tpr(self, TP, FN): + if TP + FN == 0: return 1 - return TP / (TP + FP) + return TP / (TP + FN) def _compute_fpr(self, FP, TN): if FP + TN == 0: diff --git a/quapy/method/aggregative.py b/quapy/method/aggregative.py index 25fc1ef..f678546 100644 --- a/quapy/method/aggregative.py +++ b/quapy/method/aggregative.py @@ -1,10 +1,8 @@ +import warnings from abc import ABC, abstractmethod -from argparse import ArgumentError from copy import deepcopy from typing import Callable, Literal, Union import numpy as np -from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling -from numpy.f2py.crackfortran import true_intent_list from sklearn.base import BaseEstimator from sklearn.calibration import CalibratedClassifierCV from sklearn.exceptions import NotFittedError @@ -18,11 +16,18 @@ from quapy.functional import get_divergence from quapy.classification.svmperf import SVMperf from quapy.data import LabelledCollection from quapy.method.base import BaseQuantifier, BinaryQuantifier, OneVsAllGeneric -from quapy.method import _bayesian +from quapy.method._energy import _EnergyDistanceCore +from quapy.method._helper import ( + _get_abstention_calibrators, + _get_cvxpy, + _rlls_check_mode, + _rlls_joint_distribution, + _rlls_predicted_marginal, + _rlls_compute_3deltaC, + _rlls_compute_weights, + _labels_to_indices, +) -# import warnings -# from sklearn.exceptions import ConvergenceWarning -# warnings.filterwarnings("ignore", category=ConvergenceWarning) # Abstract classes @@ -78,9 +83,9 @@ class AggregativeQuantifier(BaseQuantifier, ABC): (f'when {val_split=} is indicated as an integer, it represents the number of folds in a kFCV ' f'and must thus be >1') if val_split==5 and not fit_classifier: - print(f'Warning: {val_split=} will be ignored when the classifier is already trained ' - f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} ' - f'to None to avoid this warning.') + warnings.warn(f'{val_split=} will be ignored when the classifier is already trained ' + f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} ' + f'to None to avoid this warning.') self.val_split=None if val_split!=5: assert fit_classifier, (f'Parameter {val_split=} has been modified, but {fit_classifier=} ' @@ -339,8 +344,8 @@ class AggregativeSoftQuantifier(AggregativeQuantifier, ABC): """ if not hasattr(self.classifier, self._classifier_method()): if adapt_if_necessary: - print(f'warning: The learner {self.classifier.__class__.__name__} does not seem to be ' - f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).') + warnings.warn(f'The learner {self.classifier.__class__.__name__} does not seem to be ' + f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).') self.classifier = CalibratedClassifierCV(self.classifier, cv=5) else: raise AssertionError(f'error: The learner {self.classifier.__class__.__name__} does not ' @@ -367,8 +372,13 @@ class BinaryAggregativeQuantifier(AggregativeQuantifier, BinaryQuantifier): # ------------------------------------ class CC(AggregativeCrispQuantifier): """ - The most basic Quantification method. One that simply classifies all instances and counts how many have been - attributed to each of the classes in order to compute class prevalence estimates. + `Classify & Count` (CC), the most basic quantification method, one that + simply classifies all instances and counts how many have been attributed to + each class in order to compute class prevalence estimates. This baseline is + the unadjusted estimator discussed, among others, in + `Forman, G. (2008). Quantifying counts and costs via classification. + Data Mining and Knowledge Discovery, 17, 164-206 + `_. :param classifier: a sklearn's Estimator that generates a classifier """ @@ -396,14 +406,19 @@ class CC(AggregativeCrispQuantifier): class PCC(AggregativeSoftQuantifier): """ - `Probabilistic Classify & Count `_, - the probabilistic variant of CC that relies on the posterior probabilities returned by a probabilistic classifier. + `Probabilistic Classify & Count` (PCC), the probabilistic variant of CC + that relies on the posterior probabilities returned by a probabilistic + classifier, introduced in + `Bella, A., Ferri, C., Hernández-Orallo, J., and Ramírez-Quintana, M.J. + (2010). Quantification via probability estimators. In Proceedings of the + 2010 IEEE International Conference on Data Mining (ICDM 2010) + `_. :param classifier: a sklearn's Estimator that generates a classifier """ - def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True): - super().__init__(classifier, fit_classifier, val_split=None) + def __init__(self, classifier: BaseEstimator = None, fit_classifier: bool = True, val_split=None): + super().__init__(classifier, fit_classifier, val_split=val_split) def aggregation_fit(self, classif_predictions, labels): """ @@ -420,9 +435,12 @@ class PCC(AggregativeSoftQuantifier): class ACC(AggregativeCrispQuantifier): """ - `Adjusted Classify & Count `_, - the "adjusted" variant of :class:`CC`, that corrects the predictions of CC - according to the `misclassification rates`. + `Adjusted Classify & Count` (ACC), the "adjusted" variant of :class:`CC` + that corrects the predictions of CC according to the + misclassification rates, originally proposed in + `Forman, G. (2008). Quantifying counts and costs via classification. + Data Mining and Knowledge Discovery, 17, 164-206 + `_. :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be the one indicated in `qp.environ['DEFAULT_CLS']` @@ -564,8 +582,13 @@ class ACC(AggregativeCrispQuantifier): class PACC(AggregativeSoftQuantifier): """ - `Probabilistic Adjusted Classify & Count `_, - the probabilistic variant of ACC that relies on the posterior probabilities returned by a probabilistic classifier. + `Probabilistic Adjusted Classify & Count` (PACC), the probabilistic + variant of ACC that relies on the posterior probabilities returned by a + probabilistic classifier, introduced in + `Bella, A., Ferri, C., Hernández-Orallo, J., and Ramírez-Quintana, M.J. + (2010). Quantification via probability estimators. In Proceedings of the + 2010 IEEE International Conference on Data Mining (ICDM 2010) + `_. :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be the one indicated in `qp.environ['DEFAULT_CLS']` @@ -670,6 +693,108 @@ class PACC(AggregativeSoftQuantifier): return confusion.T +class RLLS(AggregativeSoftQuantifier): + """ + `Regularized Learning for Domain Adaptation under Label Shifts + `_, used here as an aggregative + quantifier. + + This implementation ports the regularized weight-estimation component of + RLLS to QuaPy's aggregative interface. It estimates label-shift ratios from + validation posteriors and source labels, then rescales the source + prevalence to obtain target prevalence estimates. + + This method relies on the optional `cvxpy` dependency. + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case + the classifier is taken to be the one indicated in + `qp.environ['DEFAULT_CLS']` + :param fit_classifier: whether to train the learner (default is True). Set + to False if the learner has been trained outside the quantifier. + :param val_split: specifies the data used for generating classifier + predictions. This specification can be made as float in (0, 1) + indicating the proportion of stratified held-out validation set to be + extracted from the training set; or as an integer (default 5), + indicating that the predictions are to be generated in a `k`-fold + cross-validation manner; or as a tuple `(X, y)` defining the specific + set of data to use for validation. This method requires source + predictions and therefore needs `val_split` whenever + `fit_classifier=True`. + :param mode: whether source- and target-domain quantities are estimated + from posterior probabilities (`soft`, default) or from argmax + predictions (`hard`) + :param alpha: multiplicative factor for the regularization level (default + 0.01) + :param delta: confidence parameter used in the finite-sample regularizer + (default 0.05) + :param clip_weights: if True, clips negative importance weights to zero + before converting them into prevalence estimates + :param norm: the normalization method passed to + :func:`quapy.functional.normalize_prevalence` + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier=True, + val_split=5, + mode: Literal['soft', 'hard'] = 'soft', + alpha: float = 0.01, + delta: float = 0.05, + clip_weights: bool = True, + norm: Literal['clip', 'mapsimplex', 'condsoftmax'] = 'clip', + ): + super().__init__(classifier, fit_classifier, val_split) + self.mode = mode + self.alpha = alpha + self.delta = delta + self.clip_weights = clip_weights + self.norm = norm + self.last_w_ = None + + def _check_init_parameters(self): + _get_cvxpy() + _rlls_check_mode(self.mode) + if not isinstance(self.alpha, (int, float)) or self.alpha < 0: + raise ValueError(f'expected a non-negative real value for alpha; found {self.alpha!r}') + if not isinstance(self.delta, (int, float)) or not (0 < self.delta < 1): + raise ValueError(f'expected delta to be in (0,1); found {self.delta!r}') + if self.norm not in ACC.NORMALIZATIONS: + raise ValueError(f"unknown normalization; valid ones are {ACC.NORMALIZATIONS}") + if self.fit_classifier and self.val_split is None: + raise ValueError( + 'RLLS requires validation predictions for aggregation_fit; ' + 'please set val_split to an integer, float, or validation tuple' + ) + + def aggregation_fit(self, classif_predictions, labels): + if classif_predictions is None or labels is None: + raise ValueError('RLLS requires source posterior predictions and source labels') + + self.train_prevalence_ = F.prevalence_from_labels(labels, classes=self.classes_) + self.C_zy_ = _rlls_joint_distribution( + classif_predictions, + labels, + self.classes_, + mode=self.mode, + ) + self.pz_ = _rlls_predicted_marginal(classif_predictions, mode=self.mode) + self.rho_ = _rlls_compute_3deltaC(len(self.classes_), len(labels), self.delta) + + def aggregate(self, classif_posteriors): + qz = _rlls_predicted_marginal(classif_posteriors, mode=self.mode) + w = _rlls_compute_weights( + self.C_zy_, + qz, + self.pz_, + rho=self.alpha * self.rho_, + clip=self.clip_weights, + ) + self.last_w_ = w + estimate = self.train_prevalence_ * w + return F.normalize_prevalence(estimate, method=self.norm) + + class EMQ(AggregativeSoftQuantifier): """ `Expectation Maximization for Quantification `_ (EMQ), @@ -732,7 +857,7 @@ class EMQ(AggregativeSoftQuantifier): self.exact_train_prev = exact_train_prev self.calib = calib self.on_calib_error = on_calib_error - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) @classmethod def EMQ_BCTS(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, on_calib_error="raise", n_jobs=None): @@ -769,15 +894,15 @@ class EMQ(AggregativeSoftQuantifier): def _check_init_parameters(self): if self.val_split is not None: if self.exact_train_prev and self.calib is None: - raise RuntimeWarning(f'The parameter {self.val_split=} was specified for EMQ, while the parameters ' - f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an ' - f'unnecessary overload.') + warnings.warn(f'The parameter {self.val_split=} was specified for EMQ, while the parameters ' + f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an ' + f'unnecessary overload.', RuntimeWarning) else: if self.calib is not None: - print(f'[warning] The parameter {self.calib=} requires the val_split be different from None. ' - f'This parameter will be set to 5. To avoid this warning, set this value to a float value ' - f'indicating the proportion of training data to be used as validation, or to an integer ' - f'indicating the number of folds for kFCV.') + warnings.warn(f'The parameter {self.calib=} requires the val_split be different from None. ' + f'This parameter will be set to 5. To avoid this warning, set this value to a float value ' + f'indicating the proportion of training data to be used as validation, or to an integer ' + f'indicating the number of folds for kFCV.') self.val_split = 5 def classify(self, X): @@ -839,22 +964,17 @@ class EMQ(AggregativeSoftQuantifier): requires_predictions = (self.calib is not None) or (not self.exact_train_prev) if P is None and requires_predictions: # classifier predictions were not generated because val_split=None - raise ArgumentError(self.val_split, self.__class__.__name__ + - ": Classifier predictions for the aggregative fit were not generated because " - "val_split=None. This usually happens when you enable calibrations or heuristics " - "during model selection but left val_split set to its default value (None). " - "Please provide one of the following values for val_split: (i) an integer >1 " - "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. " - "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit " - "validation data") + raise ValueError(self.__class__.__name__ + + ": Classifier predictions for the aggregative fit were not generated because " + "val_split=None. This usually happens when you enable calibrations or heuristics " + "during model selection but left val_split set to its default value (None). " + "Please provide one of the following values for val_split: (i) an integer >1 " + "(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. " + "val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit " + "validation data") if self.calib is not None: - calibrator = { - 'nbvs': NoBiasVectorScaling(), - 'bcts': TempScaling(bias_positions='all'), - 'ts': TempScaling(), - 'vs': VectorScaling() - }.get(self.calib, None) + calibrator = _get_abstention_calibrators().get(self.calib, None) if calibrator is None: raise ValueError(f'invalid value for {self.calib=}; valid ones are {EMQ.CALIB_OPTIONS}') @@ -922,7 +1042,7 @@ class EMQ(AggregativeSoftQuantifier): s += 1 if not converged: - print('[warning] the method has reached the maximum number of iterations; it might have not converged') + warnings.warn('the method has reached the maximum number of iterations; it might have not converged') return qs, ps @@ -937,6 +1057,10 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): class-conditional distributions of the posterior probabilities returned for the positive and negative validation examples, respectively. The parameters of the mixture thus represent the estimates of the class prevalence values. + This dedicated class is kept for backward compatibility as the historical + HDy implementation. The same historical preset is also available as + :meth:`DMy.HDy`. + :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be the one indicated in `qp.environ['DEFAULT_CLS']` @@ -983,9 +1107,6 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): Px = classif_posteriors[:, self.pos_label] # takes only the P(y=+1|x) prev_estimations = [] - # for bins in np.linspace(10, 110, 11, dtype=int): #[10, 20, 30, ..., 100, 110] - # Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True) - # Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True) for bins in self.bins: Pxy0_density = self.Pxy0_density[bins] Pxy1_density = self.Pxy1_density[bins] @@ -995,13 +1116,12 @@ class HDy(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): # the authors proposed to search for the prevalence yielding the best matching as a linear search # at small steps (modern implementations resort to an optimization procedure, # see class DistributionMatching) - prev_selected, min_dist = None, None - for prev in F.prevalence_linspace(grid_points=101, repeats=1, smooth_limits_epsilon=0.0): - Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density - hdy = F.HellingerDistance(Px_train, Px_test) - if prev_selected is None or hdy < min_dist: - prev_selected, min_dist = prev, hdy - prev_estimations.append(prev_selected) + def loss(prev): + class1_prev = prev[1] + Px_train = class1_prev * Pxy1_density + (1 - class1_prev) * Pxy0_density + return F.HellingerDistance(Px_train, Px_test) + + prev_estimations.append(F.linear_search(loss, n_classes=2)[1]) class1_prev = np.median(prev_estimations) return F.as_binary_prevalence(class1_prev) @@ -1042,7 +1162,7 @@ class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): self.tol = tol self.divergence = divergence self.n_bins = n_bins - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) def _ternary_search(self, f, left, right, tol): """ @@ -1160,6 +1280,10 @@ class DMy(AggregativeSoftQuantifier): :param cdf: whether to use CDF instead of PDF (default False) + :param search: string indicating the search strategy used to estimate the prevalence values. + Valid options are `optim_minimize` (default, works for binary and multiclass problems), + `linear_search` (binary only), and `ternary_search` (binary only) + :param n_jobs: number of parallel workers (default None) """ @@ -1170,15 +1294,36 @@ class DMy(AggregativeSoftQuantifier): self.divergence = divergence self.cdf = cdf self.search = search - self.n_jobs = n_jobs + self.n_jobs = qp._get_njobs(n_jobs) - # @classmethod - # def HDy(cls, classifier, val_split=5, n_jobs=None): - # from quapy.method.meta import MedianEstimator - # - # hdy = DMy(classifier=classifier, val_split=val_split, search='linear_search', divergence='HD') - # hdy = AggregativeMedianEstimator(hdy, param_grid={'nbins': np.linspace(10, 110, 11).astype(int)}, n_jobs=n_jobs) - # return hdy + @classmethod + def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None): + """ + Historical HDy preset expressed as a configuration of :class:`DMy`. + + This preset reproduces the original HDy setup by using Hellinger + distance, PDF matching, linear search, and a median sweep over + `nbins` in `[10, 20, ..., 110]`. + + :param classifier: a scikit-learn's BaseEstimator, or None + :param fit_classifier: whether to train the learner + :param val_split: validation specification for generating posteriors + :param n_jobs: number of parallel workers + :return: an instance of :class:`AggregativeMedianEstimator` configured + to reproduce the historical HDy preset + """ + base = cls( + classifier=classifier, + fit_classifier=fit_classifier, + val_split=val_split, + nbins=10, + divergence='HD', + cdf=False, + search='linear_search', + n_jobs=n_jobs, + ) + param_grid = {'nbins': np.linspace(10, 110, 11, dtype=int)} + return AggregativeMedianEstimator(base_quantifier=base, param_grid=param_grid, n_jobs=n_jobs) def _get_distributions(self, posteriors): histograms = [] @@ -1210,7 +1355,9 @@ class DMy(AggregativeSoftQuantifier): :param labels: array-like with the true labels associated to each posterior """ posteriors, true_labels = classif_predictions, labels - n_classes = len(self.classifier.classes_) + classes = self.classifier.classes_ + n_classes = len(classes) + true_labels = _labels_to_indices(true_labels, classes) self.validation_distribution = qp.util.parallel( func=self._get_distributions, @@ -1321,9 +1468,9 @@ def newSVMKLD(svmperf_base=None, C=1): return newELM(svmperf_base, loss='kld', C=C) -def newSVMKLD(svmperf_base=None, C=1): +def newSVMNKLD(svmperf_base=None, C=1): """ - SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence + SVM(NKLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence normalized via the logistic function, as proposed by `Esuli et al. 2015 `_. Equivalent to: @@ -1450,7 +1597,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier): return F.normalize_prevalence(prevalences) def aggregation_fit(self, classif_predictions, labels): - self._parallel(self._delayed_binary_aggregate_fit(c, classif_predictions, labels)) + self._parallel(self._delayed_binary_aggregate_fit, classif_predictions, labels) return self def _delayed_binary_classification(self, c, X): @@ -1462,7 +1609,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier): def _delayed_binary_aggregate_fit(self, c, classif_predictions, labels): # trains the aggregation function of the cth quantifier - return self.dict_binary_quantifiers[c].aggregate_fit(classif_predictions[:, c], labels) + return self.dict_binary_quantifiers[c].aggregation_fit(classif_predictions[:, c], labels == c) class AggregativeMedianEstimator(BinaryQuantifier): @@ -1528,8 +1675,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): ((params, X, y) for params in cls_configs), seed=qp.environ.get('_R_SEED', None), n_jobs=self.n_jobs, - asarray=False, - backend='threading' + asarray=False ) else: model = self.base_quantifier @@ -1541,8 +1687,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_fit_aggregation, itertools.product(models_preds, q_configs), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) else: configs = qp.model_selection.expand_grid(self.param_grid) @@ -1550,8 +1695,7 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_fit, ((params, X, y) for params in configs), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) return self @@ -1564,12 +1708,111 @@ class AggregativeMedianEstimator(BinaryQuantifier): self._delayed_predict, ((model, instances) for model in self.models), seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs, - backend='threading' + n_jobs=self.n_jobs ) return np.median(prev_preds, axis=0) +class EDy(_EnergyDistanceCore, AggregativeSoftQuantifier): + """ + Energy Distance y (EDy), a posterior-space distribution-matching quantifier + based on energy distance. + + The method represents each class by the posterior-probability vectors + produced by a probabilistic classifier on validation data, and estimates the + test prevalence vector by matching the test posterior distribution against + the class-conditional validation distributions through an energy-distance + objective solved as a quadratic program. The method is therefore another + instance of the general mixture-matching view of quantification, but it + operates directly on posterior vectors rather than on histogram summaries. + + This implementation works for binary and multiclass single-label + quantification and relies on the optional ``quadprog`` dependency. It was + adapted to QuaPy's current aggregative API from the original implementation + available in `quantificationlib `_, + and now shares its numerical core with the classifier-free + :class:`quapy.method.non_aggregative.EDx` variant. + + The current implementation follows the energy-distance formulation discussed + in: + + * Alberto Castaño, Laura Morán-Fernández, Jaime Alonso, + Verónica Bolón-Canedo, Amparo Alonso-Betanzos, and Juan José del Coz. + *An analysis of quantification methods based on matching distributions*. + * Hideko Kawakubo, Marthinus Christoffel du Plessis, and Masashi Sugiyama + (2016). *Computationally efficient class-prior estimation under class + balance change using energy distance*. IEICE Transactions on Information + and Systems, 99(1):176-186. + + :param classifier: a scikit-learn ``BaseEstimator``, or ``None`` to use + ``qp.environ['DEFAULT_CLS']`` + :param fit_classifier: whether to train the learner (default ``True``). + Set to ``False`` if the learner has already been trained outside the + quantifier + :param val_split: specification of the data used for generating validation + posterior probabilities. This can be an integer (default ``5``) for + k-fold cross-validation, a float in ``(0, 1)`` for a held-out split, + or a tuple ``(X, y)`` with explicit validation data + :param distance: distance used to compare posterior vectors. Valid string + aliases are ``'manhattan'`` (default) and ``'euclidean'``; a custom + callable compatible with pairwise-distance signatures can also be used + :param n_jobs: number of parallel workers (default ``None``, meaning the + value is taken from the environment) + """ + + def __init__( + self, + classifier: BaseEstimator = None, + fit_classifier: bool = True, + val_split=5, + distance: Union[str, Callable] = 'manhattan', + n_jobs=None, + ): + super().__init__(classifier, fit_classifier, val_split) + self.distance = distance + self.n_jobs = qp._get_njobs(n_jobs) + self.train_n_cls_i_ = None + self.train_distrib_ = None + self.K_ = None + self.G_ = None + self.C_ = None + self.b_ = None + self.a_ = None + + def _check_init_parameters(self): + self._check_ed_init_parameters() + + def aggregation_fit(self, classif_predictions, labels): + """ + Estimate the class-conditional posterior distributions on validation + data and pre-compute the quadratic-program parameters that depend only + on the training side. + + In EDy, the validation posteriors are not discretized into histograms. + Instead, each class is represented by the cloud of posterior vectors + observed for that class, and these clouds are then compared through the + selected pairwise distance. + + :param classif_predictions: posterior probabilities returned by the + classifier on validation data + :param labels: true labels associated to each posterior vector + """ + posteriors = np.asarray(classif_predictions, dtype=float) + labels = np.asarray(labels) + train_distrib = [posteriors[labels == class_] for class_ in self.classes_] + return self._fit_energy_model(train_distrib) + + def aggregate(self, posteriors: np.ndarray): + """Estimate the prevalence vector for a test sample. + + :param posteriors: posterior probabilities returned by the classifier + for the instances in the test sample + :return: a prevalence vector of shape ``(n_classes,)`` + """ + posteriors = np.asarray(posteriors, dtype=float) + return self._predict_energy(posteriors) + + # --------------------------------------------------------------- # imports # --------------------------------------------------------------- @@ -1588,6 +1831,7 @@ KDEyML = _kdey.KDEyML KDEyHD = _kdey.KDEyHD KDEyCS = _kdey.KDEyCS + # --------------------------------------------------------------- # aliases # --------------------------------------------------------------- @@ -1599,6 +1843,8 @@ ProbabilisticAdjustedClassifyAndCount = PACC ExpectationMaximizationQuantifier = EMQ SLD = EMQ DistributionMatchingY = DMy +EnergyDistanceY = EDy HellingerDistanceY = HDy +HistoricalHDy = DMy.HDy MedianSweep = MS MedianSweep2 = MS2 diff --git a/quapy/method/base.py b/quapy/method/base.py index 85a0525..85337ed 100644 --- a/quapy/method/base.py +++ b/quapy/method/base.py @@ -1,3 +1,4 @@ +import warnings from abc import ABCMeta, abstractmethod from copy import deepcopy @@ -84,8 +85,8 @@ class OneVsAllGeneric(OneVsAll, BaseQuantifier): assert isinstance(binary_quantifier, BaseQuantifier), \ f'{binary_quantifier} does not seem to be a Quantifier' if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier): - print('[warning] the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; ' - f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}') + warnings.warn('the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; ' + f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}') self.binary_quantifier = binary_quantifier self.n_jobs = qp._get_njobs(n_jobs) diff --git a/quapy/method/confidence.py b/quapy/method/confidence.py index f54768c..885870b 100644 --- a/quapy/method/confidence.py +++ b/quapy/method/confidence.py @@ -1,25 +1,33 @@ +from numbers import Number +from typing import Iterable + import numpy as np +from joblib import Parallel, delayed from sklearn.base import BaseEstimator from sklearn.metrics import confusion_matrix import quapy as qp import quapy.functional as F -from quapy.method import _bayesian -from quapy.method.aggregative import AggregativeCrispQuantifier +from quapy.functional import CompositionalTransformation, CLRtransformation, ILRtransformation from quapy.data import LabelledCollection -from quapy.method.aggregative import AggregativeQuantifier +from quapy.method.aggregative import AggregativeQuantifier, AggregativeCrispQuantifier, AggregativeSoftQuantifier, BinaryAggregativeQuantifier from scipy.stats import chi2 from sklearn.utils import resample from abc import ABC, abstractmethod -from scipy.special import softmax, factorial +from scipy.special import factorial import copy -from functools import lru_cache +from tqdm import tqdm """ This module provides implementation of different types of confidence regions, and the implementation of Bootstrap for AggregativeQuantifiers. """ + +def _get_bayesian_module(): + from quapy.method import _bayesian + return _bayesian + class ConfidenceRegionABC(ABC): """ Abstract class of confidence regions @@ -55,7 +63,6 @@ class ConfidenceRegionABC(ABC): """ ... - @lru_cache def simplex_portion(self): """ Computes the fraction of the simplex which is covered by the region. This is not the volume of the region @@ -64,9 +71,10 @@ class ConfidenceRegionABC(ABC): :return: float, the fraction of the simplex covered by the region """ - return self.montecarlo_proportion() + if not hasattr(self, '_simplex_portion_cache'): + self._simplex_portion_cache = self.montecarlo_proportion() + return self._simplex_portion_cache - @lru_cache def montecarlo_proportion(self, n_trials=10_000): """ Estimates, via a Monte Carlo approach, the fraction of the simplex covered by the region. This is carried @@ -75,33 +83,100 @@ class ConfidenceRegionABC(ABC): :return: float in [0,1] """ - with qp.util.temp_seed(0): - uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials) - proportion = np.clip(self.coverage(uniform_simplex), 0., 1.) - return proportion + if not hasattr(self, '_montecarlo_proportion_cache'): + self._montecarlo_proportion_cache = {} + if n_trials not in self._montecarlo_proportion_cache: + with qp.util.temp_seed(0): + uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials) + self._montecarlo_proportion_cache[n_trials] = np.clip(self.coverage(uniform_simplex), 0., 1.) + return self._montecarlo_proportion_cache[n_trials] + + @property + @abstractmethod + def samples(self): + """ Returns internal samples """ + ... + + def __contains__(self, p): + """ + Overloads in operator, checks if `p` is contained in the region + + :param p: array-like + :return: boolean + """ + p = np.asarray(p) + assert p.ndim==1, f'unexpected shape for point parameter' + return self.coverage(p)==1. + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + """ + Finds the closes point to p that belongs to the region. Assumes the region is convex. + + :param p: array-like, the point + :param tol: float, error tolerance + :param max_iter: int, max number of iterations + :returns: array-like, the closes point to p in the segment between p and the center of the region, that + belongs to the region + """ + p = np.asarray(p, dtype=float) + + # if p in region, returns p itself + if p in self: + return p.copy() + + # center of the region + c = self.point_estimate() + + # binary search in [0,1], interpolation parameter + # low=closest to p, high=closest to c + low, high = 0.0, 1.0 + for _ in range(max_iter): + mid = 0.5 * (low + high) + x = p*(1-mid) + c*mid + if x in self: + high = mid + else: + low = mid + if high - low < tol: + break + + in_boundary = p*(1-high) + c*high + return in_boundary class WithConfidenceABC(ABC): """ Abstract class for confidence regions. """ - METHODS = ['intervals', 'ellipse', 'ellipse-clr'] + REGION_TYPE = ['intervals', 'ellipse', 'ellipse-clr', 'ellipse-ilr'] @abstractmethod - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): """ - Adds the method `quantify_conf` to the interface. This method returns not only the point-estimate, but + Adds the method `predict_conf` to the interface. This method returns not only the point-estimate, but also the confidence region around it. :param instances: a np.ndarray of shape (n_instances, n_features,) - :confidence_level: float in (0, 1) + :param confidence_level: float in (0, 1), default is 0.95 :return: a tuple (`point_estimate`, `conf_region`), where `point_estimate` is a np.ndarray of shape (n_classes,) and `conf_region` is an object from :class:`ConfidenceRegionABC` """ ... + def quantify_conf(self, instances, confidence_level=0.95) -> (np.ndarray, ConfidenceRegionABC): + """ + Alias to `predict_conf`. This method returns not only the point-estimate, but + also the confidence region around it. + + :param instances: a np.ndarray of shape (n_instances, n_features,) + :param confidence_level: float in (0, 1), default is 0.95 + :return: a tuple (`point_estimate`, `conf_region`), where `point_estimate` is a np.ndarray of shape + (n_classes,) and `conf_region` is an object from :class:`ConfidenceRegionABC` + """ + return self.predict_conf(instances=instances, confidence_level=confidence_level) + @classmethod - def construct_region(cls, prev_estims, confidence_level=0.95, method='intervals'): + def construct_region(cls, prev_estims, confidence_level=0.95, method='intervals', bonferroni=False)->ConfidenceRegionABC: """ Construct a confidence region given many prevalence estimations. @@ -111,14 +186,22 @@ class WithConfidenceABC(ABC): constructing confidence intervals (default), or to `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni correction when + `method='intervals'`. This parameter has no effect for ellipse-based regions. """ region = None if method == 'intervals': - region = ConfidenceIntervals(prev_estims, confidence_level=confidence_level) + region = ConfidenceIntervals( + prev_estims, + confidence_level=confidence_level, + bonferroni_correction=bonferroni, + ) elif method == 'ellipse': region = ConfidenceEllipseSimplex(prev_estims, confidence_level=confidence_level) elif method == 'ellipse-clr': region = ConfidenceEllipseCLR(prev_estims, confidence_level=confidence_level) + elif method == 'ellipse-ilr': + region = ConfidenceEllipseILR(prev_estims, confidence_level=confidence_level) if region is None: raise NotImplementedError(f'unknown method {method}') @@ -136,7 +219,7 @@ def simplex_volume(n): return 1 / factorial(n) -def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): +def within_ellipse_prop__(values, mean, prec_matrix, chi2_critical): """ Checks the proportion of values that belong to the ellipse with center `mean` and precision matrix `prec_matrix` at a distance `chi2_critical`. @@ -169,110 +252,127 @@ def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): return within_elipse * 1.0 -class ConfidenceEllipseSimplex(ConfidenceRegionABC): +def within_ellipse_prop(values, mean, prec_matrix, chi2_critical): """ - Instantiates a Confidence Ellipse in the probability simplex. + Checks the proportion of values that belong to the ellipse with center `mean` and precision matrix `prec_matrix` + at a distance `chi2_critical`. - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) - :param confidence_level: float, the confidence level (default 0.95) + :param values: a np.ndarray of shape (n_dim,) or (n_values, n_dim,) + :param mean: a np.ndarray of shape (n_dim,) with the center of the ellipse + :param prec_matrix: a np.ndarray with the precision matrix (inverse of the + covariance matrix) of the ellipse. If this inverse cannot be computed + then None must be passed + :param chi2_critical: float, the chi2 critical value + + :return: float in [0,1], the fraction of values that are contained in the ellipse + defined by the mean (center), the precision matrix (shape), and the chi2_critical value (distance). + If `values` is only one value, then either 0. (not contained) or 1. (contained) is returned. + """ + if prec_matrix is None: + return 0. + + values = np.atleast_2d(values) + diff = values - mean + d_M_squared = np.sum(diff @ prec_matrix * diff, axis=-1) + within_ellipse = d_M_squared <= chi2_critical + + if len(within_ellipse) == 1: + return float(within_ellipse[0]) + else: + return float(np.mean(within_ellipse)) + + +def closest_point_on_ellipsoid(p, mean, cov, chi2_critical, tol=1e-9, max_iter=100): + """ + Computes the closest point on the ellipsoid defined by: + (x - mean)^T cov^{-1} (x - mean) = chi2_critical """ - def __init__(self, X, confidence_level=0.95): + p = np.asarray(p) + mean = np.asarray(mean) + Sigma = np.asarray(cov) - assert 0. < confidence_level < 1., f'{confidence_level=} must be in range(0,1)' + # Precompute precision matrix + P = np.linalg.pinv(Sigma) + d = P.shape[0] - X = np.asarray(X) + # Define v = p - mean + v = p - mean - self.mean_ = X.mean(axis=0) - self.cov_ = np.cov(X, rowvar=False, ddof=1) + # If p is inside the ellipsoid, return p itself + M_dist = v @ P @ v + if M_dist <= chi2_critical: + return p.copy() - try: - self.precision_matrix_ = np.linalg.inv(self.cov_) - except: - self.precision_matrix_ = None + # Function to compute x(lambda) + def x_lambda(lmbda): + A = np.eye(d) + lmbda * P + return mean + np.linalg.solve(A, v) - self.dim = X.shape[-1] - self.ddof = self.dim - 1 + # Function whose root we want: f(lambda) = Mahalanobis distance - chi2 + def f(lmbda): + x = x_lambda(lmbda) + diff = x - mean + return diff @ P @ diff - chi2_critical - # critical chi-square value - self.confidence_level = confidence_level - self.chi2_critical_ = chi2.ppf(confidence_level, df=self.ddof) + # Bisection search over lambda >= 0 + l_low, l_high = 0.0, 1.0 - def point_estimate(self): - """ - Returns the point estimate, the center of the ellipse. + # Increase high until f(high) < 0 + while f(l_high) > 0: + l_high *= 2 + if l_high > 1e12: + raise RuntimeError("Failed to bracket the root.") - :return: np.ndarray of shape (n_classes,) - """ - return self.mean_ + # Bisection + for _ in range(max_iter): + l_mid = 0.5 * (l_low + l_high) + fm = f(l_mid) + if abs(fm) < tol: + break + if fm > 0: + l_low = l_mid + else: + l_high = l_mid - def coverage(self, true_value): - """ - Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the - fraction of these that are contained in the region, if more than one value is passed. If only one value is - passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. - - :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) - :return: float in [0,1] - """ - return within_ellipse_prop(true_value, self.mean_, self.precision_matrix_, self.chi2_critical_) - - -class ConfidenceEllipseCLR(ConfidenceRegionABC): - """ - Instantiates a Confidence Ellipse in the Centered-Log Ratio (CLR) space. - - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) - :param confidence_level: float, the confidence level (default 0.95) - """ - - def __init__(self, X, confidence_level=0.95): - self.clr = CLRtransformation() - Z = self.clr(X) - self.mean_ = np.mean(X, axis=0) - self.conf_region_clr = ConfidenceEllipseSimplex(Z, confidence_level=confidence_level) - - def point_estimate(self): - """ - Returns the point estimate, the center of the ellipse. - - :return: np.ndarray of shape (n_classes,) - """ - # The inverse of the CLR does not coincide with the true mean, because the geometric mean - # requires smoothing the prevalence vectors and this affects the softmax (inverse); - # return self.clr.inverse(self.mean_) # <- does not coincide - return self.mean_ - - def coverage(self, true_value): - """ - Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the - fraction of these that are contained in the region, if more than one value is passed. If only one value is - passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. - - :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) - :return: float in [0,1] - """ - transformed_values = self.clr(true_value) - return self.conf_region_clr.coverage(transformed_values) + l_opt = l_mid + return x_lambda(l_opt) class ConfidenceIntervals(ConfidenceRegionABC): """ Instantiates a region based on (independent) Confidence Intervals. - :param X: np.ndarray of shape (n_bootstrap_samples, n_classes) + :param samples: np.ndarray of shape (n_samples, n_classes) :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. """ - def __init__(self, X, confidence_level=0.95): + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): assert 0 < confidence_level < 1, f'{confidence_level=} must be in range(0,1)' + assert samples.ndim == 2, 'unexpected shape; must be (n_bootstrap_samples, n_classes)' - X = np.asarray(X) + samples = np.asarray(samples) - self.means_ = X.mean(axis=0) + self.means_ = samples.mean(axis=0) + self.confidence_level = confidence_level alpha = 1-confidence_level + if bonferroni_correction: + n_classes = samples.shape[-1] + if n_classes>2: + alpha = alpha/n_classes low_perc = (alpha/2.)*100 high_perc = (1-alpha/2.)*100 - self.I_low, self.I_high = np.percentile(X, q=[low_perc, high_perc], axis=0) + self.I_low, self.I_high = np.percentile(samples, q=[low_perc, high_perc], axis=0) + self._samples = samples + self.alpha = alpha + + @property + def samples(self): + return self._samples def point_estimate(self): """ @@ -297,33 +397,278 @@ class ConfidenceIntervals(ConfidenceRegionABC): return proportion + def coverage_soft(self, true_value): + within_intervals = np.logical_and(self.I_low <= true_value, true_value <= self.I_high) + return np.mean(within_intervals.astype(float)) -class CLRtransformation: + def __repr__(self): + return '['+', '.join(f'({low:.4f}, {high:.4f})' for (low,high) in zip(self.I_low, self.I_high))+']' + + @property + def n_dim(self): + return len(self.I_low) + + def winkler_scores(self, true_prev, alpha=None, add_ae=False): + true_prev = np.asarray(true_prev) + assert true_prev.ndim == 1, 'unexpected dimensionality for true_prev' + assert len(true_prev)==self.n_dim, \ + f'unexpected number of dimensions; found {true_prev.ndim}, expected {self.n_dim}' + + def winkler_score(low, high, true_val, alpha, center): + amp = high-low + scale_cost = 2./alpha + cost = np.max([0, low-true_val], axis=0) + np.max([0, true_val-high], axis=0) + err = 0 + if add_ae: + err = abs(true_val - center) + return amp + scale_cost*cost + err + + alpha = alpha or self.alpha + return np.asarray( + [winkler_score(low_i, high_i, true_v, alpha, center) + for (low_i, high_i, true_v, center) in zip(self.I_low, self.I_high, true_prev, self.point_estimate())] + ) + + def mean_winkler_score(self, true_prev, alpha=None, add_ae=False): + return np.mean(self.winkler_scores(true_prev, alpha=alpha, add_ae=add_ae)) + + + +class ConfidenceEllipseSimplex(ConfidenceRegionABC): """ - Centered log-ratio, from component analysis + Instantiates a Confidence Ellipse in the probability simplex. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) """ - def __call__(self, X, epsilon=1e-6): - """ - Applies the CLR function to X thus mapping the instances, which are contained in `\\mathcal{R}^{n}` but - actually lie on a `\\mathcal{R}^{n-1}` simplex, onto an unrestricted space in :math:`\\mathcal{R}^{n}` - :param X: np.ndarray of (n_instances, n_dimensions) to be transformed - :param epsilon: small float for prevalence smoothing - :return: np.ndarray of (n_instances, n_dimensions), the CLR-transformed points - """ - X = np.asarray(X) - X = qp.error.smooth(X, epsilon) - G = np.exp(np.mean(np.log(X), axis=-1, keepdims=True)) # geometric mean - return np.log(X / G) + def __init__(self, samples, confidence_level=0.95): - def inverse(self, X): - """ - Inverse function. However, clr.inverse(clr(X)) does not exactly coincide with X due to smoothing. + assert 0. < confidence_level < 1., f'{confidence_level=} must be in range(0,1)' - :param X: np.ndarray of (n_instances, n_dimensions) to be transformed - :return: np.ndarray of (n_instances, n_dimensions), the CLR-transformed points + samples = np.asarray(samples) + + self.confidence_level = confidence_level + self.mean_ = samples.mean(axis=0) + self.cov_ = np.cov(samples, rowvar=False, ddof=1) + + try: + self.precision_matrix_ = np.linalg.pinv(self.cov_) + except np.linalg.LinAlgError: + self.precision_matrix_ = None + + self.dim = samples.shape[-1] + self.ddof = self.dim - 1 + + # critical chi-square value + self.confidence_level = confidence_level + self.chi2_critical_ = chi2.ppf(confidence_level, df=self.ddof) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): """ - return softmax(X, axis=-1) + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + return within_ellipse_prop(true_value, self.mean_, self.precision_matrix_, self.chi2_critical_) + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + return closest_point_on_ellipsoid( + p, + mean=self.mean_, + cov=self.cov_, + chi2_critical=self.chi2_critical_ + ) + + +class ConfidenceEllipseTransformed(ConfidenceRegionABC): + """ + Instantiates a Confidence Ellipse in a transformed space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + + def __init__(self, samples, transformation: CompositionalTransformation, confidence_level=0.95): + samples = np.asarray(samples) + self.transformation = transformation + self.confidence_level = confidence_level + Z = self.transformation(samples) + self.mean_ = np.mean(samples, axis=0) + # self.mean_ = self.transformation.inverse(np.mean(Z, axis=0)) + self.conf_region_z = ConfidenceEllipseSimplex(Z, confidence_level=confidence_level) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): + """ + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + # The inverse of the CLR does not coincide with the true mean, because the geometric mean + # requires smoothing the prevalence vectors and this affects the softmax (inverse); + # return self.clr.inverse(self.mean_) # <- does not coincide + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage(transformed_values) + + def closest_point_in_region(self, p, tol=1e-6, max_iter=30): + p_prime = self.transformation(p) + b_prime = self.conf_region_z.closest_point_in_region(p_prime, tol=tol, max_iter=max_iter) + b = self.transformation.inverse(b_prime) + return b + + +class ConfidenceEllipseCLR(ConfidenceEllipseTransformed): + """ + Instantiates a Confidence Ellipse in the Centered-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + def __init__(self, samples, confidence_level=0.95): + super().__init__(samples, CLRtransformation(), confidence_level=confidence_level) + + +class ConfidenceEllipseILR(ConfidenceEllipseTransformed): + """ + Instantiates a Confidence Ellipse in the Isometric-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + """ + def __init__(self, samples, confidence_level=0.95): + super().__init__(samples, ILRtransformation(), confidence_level=confidence_level) + + + +class ConfidenceIntervalsTransformed(ConfidenceRegionABC): + """ + Instantiates a Confidence Interval region in a transformed space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + + def __init__(self, samples, transformation: CompositionalTransformation, confidence_level=0.95, bonferroni_correction=False): + samples = np.asarray(samples) + self.transformation = transformation + self.confidence_level = confidence_level + Z = self.transformation(samples) + self.mean_ = np.mean(samples, axis=0) + # self.mean_ = self.transformation.inverse(np.mean(Z, axis=0)) + self.conf_region_z = ConfidenceIntervals(Z, confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + self._samples = samples + self.alpha = 1.-confidence_level + + @property + def samples(self): + return self._samples + + def point_estimate(self): + """ + Returns the point estimate, the center of the ellipse. + + :return: np.ndarray of shape (n_classes,) + """ + # The inverse of the CLR does not coincide with the true mean, because the geometric mean + # requires smoothing the prevalence vectors and this affects the softmax (inverse); + # return self.clr.inverse(self.mean_) # <- does not coincide + return self.mean_ + + def coverage(self, true_value): + """ + Checks whether a value, or a sets of values, are contained in the confidence region. The method computes the + fraction of these that are contained in the region, if more than one value is passed. If only one value is + passed, then it either returns 1.0 or 0.0, for indicating the value is in the region or not, respectively. + + :param true_value: a np.ndarray of shape (n_classes,) or shape (n_values, n_classes,) + :return: float in [0,1] + """ + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage(transformed_values) + + def coverage_soft(self, true_value): + transformed_values = self.transformation(true_value) + return self.conf_region_z.coverage_soft(transformed_values) + + def winkler_scores(self, true_prev, alpha=None, add_ae=False): + transformed_values = self.transformation(true_prev) + return self.conf_region_z.winkler_scores(transformed_values, alpha=alpha, add_ae=add_ae) + + def mean_winkler_score(self, true_prev, alpha=None, add_ae=False): + transformed_values = self.transformation(true_prev) + return self.conf_region_z.mean_winkler_score(transformed_values, alpha=alpha, add_ae=add_ae) + + +class ConfidenceIntervalsCLR(ConfidenceIntervalsTransformed): + """ + Instantiates a Confidence Intervals in the Centered-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): + super().__init__(samples, CLRtransformation(), confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + + +class ConfidenceIntervalsILR(ConfidenceIntervalsTransformed): + """ + Instantiates a Confidence Intervals in the Isometric-Log Ratio (CLR) space. + + :param samples: np.ndarray of shape (n_samples, n_classes) + :param confidence_level: float, the confidence level (default 0.95) + :param bonferroni_correction: bool (default False), if True, a Bonferroni correction + is applied to the significance level (`alpha`) before computing confidence intervals. + The correction consists of replacing `alpha` with `alpha/n_classes`. When + `n_classes=2` the correction is not applied because there is only one verification test + since the other class is constrained. This is not necessarily true for n_classes>2. + """ + def __init__(self, samples, confidence_level=0.95, bonferroni_correction=False): + super().__init__(samples, ILRtransformation(), confidence_level=confidence_level, bonferroni_correction=bonferroni_correction) + class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): @@ -339,6 +684,12 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): During inference, the bootstrap repetitions are applied to the pre-classified test instances. + See + `Moreo, A., Salvati, N. + An Efficient Method for Deriving Confidence Intervals in Aggregative Quantification. + Learning to Quantify: Methods and Applications (LQ 2025), co-located at ECML-PKDD 2025. + pp 12-33 `_ + :param quantifier: an aggregative quantifier :para n_train_samples: int, the number of training resamplings (defaults to 1, set to > 1 to activate a model-based bootstrap approach) @@ -348,6 +699,8 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): :param region: string, set to `intervals` for constructing confidence intervals (default), or to `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni correction when + `region='intervals'`. This parameter has no effect for ellipse-based regions. :param random_state: int for replicating samples, None (default) for non-replicable samples """ @@ -357,7 +710,9 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): n_test_samples=500, confidence_level=0.95, region='intervals', - random_state=None): + bonferroni=False, + random_state=None, + verbose=False): assert isinstance(quantifier, AggregativeQuantifier), \ f'base quantifier does not seem to be an instance of {AggregativeQuantifier.__name__}' @@ -373,33 +728,47 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): self.n_test_samples = n_test_samples self.confidence_level = confidence_level self.region = region + self.bonferroni = bonferroni self.random_state = random_state + self.verbose = verbose def aggregation_fit(self, classif_predictions, labels): - data = LabelledCollection(classif_predictions, labels, classes=self.classes_) + self.quantifiers = [] if self.n_train_samples==1: self.quantifier.aggregation_fit(classif_predictions, labels) self.quantifiers.append(self.quantifier) else: - # model-based bootstrap (only on the aggregative part) - n_examples = len(data) - full_index = np.arange(n_examples) - with qp.util.temp_seed(self.random_state): - for i in range(self.n_train_samples): - quantifier = copy.deepcopy(self.quantifier) - index = resample(full_index, n_samples=n_examples) - classif_predictions_i = classif_predictions.sampling_from_index(index) - data_i = data.sampling_from_index(index) - quantifier.aggregation_fit(classif_predictions_i, data_i) - self.quantifiers.append(quantifier) + if classif_predictions is None or labels is None: + # The entire dataset was consumed for classifier training, implying there is no need for training + # an aggregation function. If the bootstrap method was configured to train different aggregators + # (i.e., self.n_train_samples>1), then an error is raise. Otherwise, the method ends. + if self.n_train_samples > 1: + raise ValueError( + f'The underlying quantifier ({self.quantifier.__class__.__name__}) has consumed, all training ' + f'data, meaning the aggregation function needs none, but {self.n_train_samples=} is > 1, which ' + f'is inconsistent.' + ) + else: + # model-based bootstrap (only on the aggregative part) + data = LabelledCollection(classif_predictions, labels, classes=self.classes_) + n_examples = len(data) + full_index = np.arange(n_examples) + with qp.util.temp_seed(self.random_state): + for i in range(self.n_train_samples): + quantifier = copy.deepcopy(self.quantifier) + index = resample(full_index, n_samples=n_examples) + classif_predictions_i = classif_predictions.sampling_from_index(index) + data_i = data.sampling_from_index(index) + quantifier.aggregation_fit(classif_predictions_i, data_i) + self.quantifiers.append(quantifier) return self def aggregate(self, classif_predictions: np.ndarray): prev_mean, self.confidence = self.aggregate_conf(classif_predictions) return prev_mean - def aggregate_conf(self, classif_predictions: np.ndarray, confidence_level=None): + def aggregate_conf_sequential__(self, classif_predictions: np.ndarray, confidence_level=None): if confidence_level is None: confidence_level = self.confidence_level @@ -407,12 +776,32 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): prevs = [] with qp.util.temp_seed(self.random_state): for quantifier in self.quantifiers: - for i in range(self.n_test_samples): + for i in tqdm(range(self.n_test_samples), desc='resampling', total=self.n_test_samples, disable=not self.verbose): sample_i = resample(classif_predictions, n_samples=n_samples) prev_i = quantifier.aggregate(sample_i) prevs.append(prev_i) - conf = WithConfidenceABC.construct_region(prevs, confidence_level, method=self.region) + conf = WithConfidenceABC.construct_region(prevs, confidence_level, method=self.region, bonferroni=self.bonferroni) + prev_estim = conf.point_estimate() + + return prev_estim, conf + + def aggregate_conf(self, classif_predictions: np.ndarray, confidence_level=None): + confidence_level = confidence_level or self.confidence_level + + + n_samples = classif_predictions.shape[0] + prevs = [] + with qp.util.temp_seed(self.random_state): + for quantifier in self.quantifiers: + results = Parallel(n_jobs=-1)( + delayed(bootstrap_once)(i, classif_predictions, quantifier, n_samples) + for i in range(self.n_test_samples) + ) + prevs.extend(results) + + prevs = np.array(prevs) + conf = WithConfidenceABC.construct_region(prevs, confidence_level, method=self.region, bonferroni=self.bonferroni) prev_estim = conf.point_estimate() return prev_estim, conf @@ -423,7 +812,7 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): self.aggregation_fit(classif_predictions, labels) return self - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): predictions = self.quantifier.classify(instances) return self.aggregate_conf(predictions, confidence_level=confidence_level) @@ -435,9 +824,16 @@ class AggregativeBootstrap(WithConfidenceABC, AggregativeQuantifier): return self.quantifier._classifier_method() +def bootstrap_once(i, classif_predictions, quantifier, n_samples): + idx = np.random.randint(0, len(classif_predictions), n_samples) + sample = classif_predictions[idx] + prev = quantifier.aggregate(sample) + return prev + + class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): """ - `Bayesian quantification `_ method, + `Bayesian quantification `_ method (by Albert Ziegler and Paweł Czyż), which is a variant of :class:`ACC` that calculates the posterior probability distribution over the prevalence vectors, rather than providing a point estimate obtained by matrix inversion. @@ -464,6 +860,10 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): :param region: string, set to `intervals` for constructing confidence intervals (default), or to `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni correction when + `region='intervals'`. This parameter has no effect for ellipse-based regions. + :param prior: an array-like with the alpha parameters of a Dirichlet prior, a scalar real value + to be broadcast to all classes, or the string 'uniform' for a uniform, uninformative prior (default) """ def __init__(self, classifier: BaseEstimator=None, @@ -473,14 +873,22 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): num_samples: int = 1_000, mcmc_seed: int = 0, confidence_level: float = 0.95, - region: str = 'intervals'): + region: str = 'intervals', + bonferroni: bool = False, + temperature = 1., + prior = 'uniform'): if num_warmup <= 0: raise ValueError(f'parameter {num_warmup=} must be a positive integer') if num_samples <= 0: raise ValueError(f'parameter {num_samples=} must be a positive integer') + assert ((isinstance(prior, str) and prior == 'uniform') or + isinstance(prior, Number) or + (isinstance(prior, Iterable) and all(isinstance(v, Number) for v in prior))), \ + f'wrong type for {prior=}; expected "uniform", a real scalar, or an array-like of real values' - if _bayesian.DEPENDENCIES_INSTALLED is False: + bayesian = _get_bayesian_module() + if bayesian.DEPENDENCIES_INSTALLED is False: raise ImportError("Auxiliary dependencies are required. " "Run `$ pip install quapy[bayes]` to install them.") @@ -490,6 +898,9 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): self.mcmc_seed = mcmc_seed self.confidence_level = confidence_level self.region = region + self.bonferroni = bonferroni + self.temperature = temperature + self.prior = prior # Array of shape (n_classes, n_predicted_classes,) where entry (y, c) is the number of instances # labeled as class y and predicted as class c. @@ -520,11 +931,26 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): n_c_unlabeled = F.counts_from_labels(classif_predictions, self.classifier.classes_).astype(float) - self._samples = _bayesian.sample_posterior( + n_classes = len(self.classifier.classes_) + if isinstance(self.prior, str) and self.prior == 'uniform': + alpha = np.ones(n_classes, dtype=float) + elif isinstance(self.prior, Number): + alpha = np.full(n_classes, float(self.prior), dtype=float) + else: + alpha = np.asarray(self.prior, dtype=float) + if alpha.ndim != 1 or len(alpha) != n_classes: + raise ValueError( + f'wrong shape for prior; expected {n_classes} values, found shape {alpha.shape}' + ) + + bayesian = _get_bayesian_module() + self._samples = bayesian.sample_posterior_bayesianCC( n_c_unlabeled=n_c_unlabeled, n_y_and_c_labeled=self._n_and_c_labeled, num_warmup=self.num_warmup, num_samples=self.num_samples, + alpha=alpha, + temperature=self.temperature, seed=self.mcmc_seed, ) return self._samples @@ -532,20 +958,133 @@ class BayesianCC(AggregativeCrispQuantifier, WithConfidenceABC): def get_prevalence_samples(self): if self._samples is None: raise ValueError("sample_from_posterior must be called before get_prevalence_samples") - return self._samples[_bayesian.P_TEST_Y] + bayesian = _get_bayesian_module() + return self._samples[bayesian.P_TEST_Y] def get_conditional_probability_samples(self): if self._samples is None: raise ValueError("sample_from_posterior must be called before get_conditional_probability_samples") - return self._samples[_bayesian.P_C_COND_Y] + bayesian = _get_bayesian_module() + return self._samples[bayesian.P_C_COND_Y] def aggregate(self, classif_predictions): - samples = self.sample_from_posterior(classif_predictions)[_bayesian.P_TEST_Y] + bayesian = _get_bayesian_module() + samples = self.sample_from_posterior(classif_predictions)[bayesian.P_TEST_Y] return np.asarray(samples.mean(axis=0), dtype=float) - def quantify_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + if confidence_level is None: + confidence_level = self.confidence_level classif_predictions = self.classify(instances) point_estimate = self.aggregate(classif_predictions) samples = self.get_prevalence_samples() # available after calling "aggregate" function - region = WithConfidenceABC.construct_region(samples, confidence_level=self.confidence_level, method=self.region) + region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region, bonferroni=self.bonferroni) return point_estimate, region + + +class PQ(AggregativeSoftQuantifier, BinaryAggregativeQuantifier): + """ + `Precise Quantifier: Bayesian distribution matching quantifier , + which is a variant of :class:`HDy` that calculates the posterior probability distribution + over the prevalence vectors, rather than providing a point estimate. + + This method relies on extra dependencies, which have to be installed via: + `$ pip install quapy[bayes]` + + :param classifier: a scikit-learn's BaseEstimator, or None, in which case the classifier is taken to be + the one indicated in `qp.environ['DEFAULT_CLS']` + :param val_split: specifies the data used for generating classifier predictions. This specification + can be made as float in (0, 1) indicating the proportion of stratified held-out validation set to + be extracted from the training set; or as an integer (default 5), indicating that the predictions + are to be generated in a `k`-fold cross-validation manner (with this integer indicating the value + for `k`); or as a tuple `(X,y)` defining the specific set of data to use for validation. Set to + None when the method does not require any validation data, in order to avoid that some portion of + the training data be wasted. + :param num_warmup: number of warmup iterations for the STAN sampler (default 500) + :param num_samples: number of samples to draw from the posterior (default 1000) + :param stan_seed: random seed for the STAN sampler (default 0) + :param region: string, set to `intervals` for constructing confidence intervals (default), or to + `ellipse` for constructing an ellipse in the probability simplex, or to `ellipse-clr` for + constructing an ellipse in the Centered-Log Ratio (CLR) unconstrained space. + :param bonferroni: bool (default False), whether to apply Bonferroni correction when + `region='intervals'`. This parameter has no effect for ellipse-based regions. + """ + def __init__(self, + classifier: BaseEstimator=None, + fit_classifier=True, + val_split: int = 5, + nbins: int = 4, + fixed_bins: bool = False, + num_warmup: int = 500, + num_samples: int = 1_000, + stan_seed: int = 0, + confidence_level: float = 0.95, + region: str = 'intervals', + bonferroni: bool = False): + + if num_warmup <= 0: + raise ValueError(f'parameter {num_warmup=} must be a positive integer') + if num_samples <= 0: + raise ValueError(f'parameter {num_samples=} must be a positive integer') + + bayesian = _get_bayesian_module() + if not bayesian.DEPENDENCIES_INSTALLED: + raise ImportError("Auxiliary dependencies are required. " + "Run `$ pip install quapy[bayes]` to install them.") + + super().__init__(classifier, fit_classifier, val_split) + + self.nbins = nbins + self.fixed_bins = fixed_bins + self.num_warmup = num_warmup + self.num_samples = num_samples + self.stan_seed = stan_seed + self.stan_code = bayesian.load_stan_file() + self.confidence_level = confidence_level + self.region = region + self.bonferroni = bonferroni + + def aggregation_fit(self, classif_predictions, labels): + y_pred = classif_predictions[:, self.pos_label] + + # Compute bin limits + if self.fixed_bins: + # Uniform bins in [0,1] + self.bin_limits = np.linspace(0, 1, self.nbins + 1) + else: + # Quantile bins + self.bin_limits = np.quantile(y_pred, np.linspace(0, 1, self.nbins + 1)) + + # Assign each prediction to a bin + bin_indices = np.digitize(y_pred, self.bin_limits[1:-1], right=True) + + # Positive and negative masks + pos_mask = (labels == self.pos_label) + neg_mask = ~pos_mask + + # Count positives and negatives per bin + self.pos_hist = np.bincount(bin_indices[pos_mask], minlength=self.nbins) + self.neg_hist = np.bincount(bin_indices[neg_mask], minlength=self.nbins) + + def aggregate(self, classif_predictions): + Px_test = classif_predictions[:, self.pos_label] + test_hist, _ = np.histogram(Px_test, bins=self.bin_limits) + bayesian = _get_bayesian_module() + prevs = bayesian.pq_stan( + self.stan_code, self.nbins, self.pos_hist, self.neg_hist, test_hist, + self.num_samples, self.num_warmup, self.stan_seed + ).flatten() + self.prev_distribution = np.vstack([1-prevs, prevs]).T + return self.prev_distribution.mean(axis=0) + + def aggregate_conf(self, predictions, confidence_level=None): + if confidence_level is None: + confidence_level = self.confidence_level + point_estimate = self.aggregate(predictions) + samples = self.prev_distribution + region = WithConfidenceABC.construct_region(samples, confidence_level=confidence_level, method=self.region, bonferroni=self.bonferroni) + return point_estimate, region + + def predict_conf(self, instances, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + predictions = self.classify(instances) + return self.aggregate_conf(predictions, confidence_level=confidence_level) diff --git a/quapy/method/meta.py b/quapy/method/meta.py index 37749e1..4f869e5 100644 --- a/quapy/method/meta.py +++ b/quapy/method/meta.py @@ -1,4 +1,5 @@ import itertools +import logging from copy import deepcopy from typing import Union, List import numpy as np @@ -26,65 +27,6 @@ else: QuaNet = "QuaNet is not available due to missing torch package" -class MedianEstimator2(BinaryQuantifier): - """ - This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the - estimation returned by differently (hyper)parameterized base quantifiers. - The median of unit-vectors is only guaranteed to be a unit-vector for n=2 dimensions, - i.e., in cases of binary quantification. - - :param base_quantifier: the base, binary quantifier - :param random_state: a seed to be set before fitting any base quantifier (default None) - :param param_grid: the grid or parameters towards which the median will be computed - :param n_jobs: number of parllel workes - """ - def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None): - self.base_quantifier = base_quantifier - self.param_grid = param_grid - self.random_state = random_state - self.n_jobs = qp._get_njobs(n_jobs) - - def get_params(self, deep=True): - return self.base_quantifier.get_params(deep) - - def set_params(self, **params): - self.base_quantifier.set_params(**params) - - def _delayed_fit(self, args): - with qp.util.temp_seed(self.random_state): - params, X, y = args - model = deepcopy(self.base_quantifier) - model.set_params(**params) - model.fit(X, y) - return model - - def fit(self, X, y): - self._check_binary(y, self.__class__.__name__) - - configs = qp.model_selection.expand_grid(self.param_grid) - self.models = qp.util.parallel( - self._delayed_fit, - ((params, X, y) for params in configs), - seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs - ) - return self - - def _delayed_predict(self, args): - model, instances = args - return model.predict(instances) - - def predict(self, X): - prev_preds = qp.util.parallel( - self._delayed_predict, - ((model, X) for model in self.models), - seed=qp.environ.get('_R_SEED', None), - n_jobs=self.n_jobs - ) - prev_preds = np.asarray(prev_preds) - return np.median(prev_preds, axis=0) - - class MedianEstimator(BinaryQuantifier): """ This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the @@ -213,7 +155,7 @@ class Ensemble(BaseQuantifier): def _sout(self, msg): if self.verbose: - print('[Ensemble]' + msg) + logging.getLogger(__name__).info('[Ensemble] ' + msg) def fit(self, X, y): @@ -402,7 +344,7 @@ def _select_k(elements, order, k): def _delayed_new_instance(args): base_quantifier, data, val_split, prev, posteriors, keep_samples, verbose, sample_size = args if verbose: - print(f'\tfit-start for prev {F.strprev(prev)}, sample_size={sample_size}') + logging.getLogger(__name__).info(f'fit-start for prev {F.strprev(prev)}, sample_size={sample_size}') model = deepcopy(base_quantifier) if val_split is not None: @@ -422,7 +364,7 @@ def _delayed_new_instance(args): tr_distribution = get_probability_distribution(posteriors[sample_index]) if (posteriors is not None) else None if verbose: - print(f'\t--fit-ended for prev {F.strprev(prev)}') + logging.getLogger(__name__).info(f'fit-ended for prev {F.strprev(prev)}') return (model, tr_prevalence, tr_distribution, sample if keep_samples else None) diff --git a/quapy/method/non_aggregative.py b/quapy/method/non_aggregative.py index eff2283..0250852 100644 --- a/quapy/method/non_aggregative.py +++ b/quapy/method/non_aggregative.py @@ -1,11 +1,20 @@ -from typing import Union, Callable +from itertools import product +from tqdm import tqdm +from typing import Union, Callable, Counter import numpy as np from sklearn.feature_extraction.text import CountVectorizer +from sklearn.utils import resample +from sklearn.preprocessing import normalize +from quapy.method.confidence import WithConfidenceABC, ConfidenceRegionABC from quapy.functional import get_divergence -from quapy.data import LabelledCollection from quapy.method.base import BaseQuantifier, BinaryQuantifier +from quapy.method._helper import _labels_to_indices +from quapy.method._energy import _EnergyDistanceCore import quapy.functional as F +from scipy.optimize import lsq_linear +from scipy import sparse +import quapy as qp class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier): @@ -52,6 +61,9 @@ class DMx(BaseQuantifier): or a callable function taking two ndarrays of the same dimension as input (default "HD", meaning Hellinger Distance) :param cdf: whether to use CDF instead of PDF (default False) + :param search: string indicating the search strategy used to estimate the prevalence values. + Valid options are `optim_minimize` (default, works for binary and multiclass problems), + `linear_search` (binary only), and `ternary_search` (binary only) :param n_jobs: number of parallel workers (default None) """ @@ -87,7 +99,6 @@ class DMx(BaseQuantifier): return hdx def __get_distributions(self, X): - histograms = [] for feat_idx in range(self.nfeats): feature = X[:, feat_idx] @@ -116,7 +127,9 @@ class DMx(BaseQuantifier): """ self.nfeats = X.shape[1] self.feat_ranges = _get_features_range(X) - n_classes = len(np.unique(y)) + classes = np.unique(y) + y = _labels_to_indices(y, classes) + n_classes = len(classes) self.validation_distribution = np.asarray( [self.__get_distributions(X[y==cat]) for cat in range(n_classes)] @@ -149,53 +162,231 @@ class DMx(BaseQuantifier): return F.argmin_prevalence(loss, n_classes, method=self.search) -# class ReadMe(BaseQuantifier): -# -# def __init__(self, bootstrap_trials=100, bootstrap_range=100, bagging_trials=100, bagging_range=25, **vectorizer_kwargs): -# raise NotImplementedError('under development ...') -# self.bootstrap_trials = bootstrap_trials -# self.bootstrap_range = bootstrap_range -# self.bagging_trials = bagging_trials -# self.bagging_range = bagging_range -# self.vectorizer_kwargs = vectorizer_kwargs -# -# def fit(self, data: LabelledCollection): -# X, y = data.Xy -# self.vectorizer = CountVectorizer(binary=True, **self.vectorizer_kwargs) -# X = self.vectorizer.fit_transform(X) -# self.class_conditional_X = {i: X[y==i] for i in range(data.classes_)} -# -# def predict(self, X): -# X = self.vectorizer.transform(X) -# -# # number of features -# num_docs, num_feats = X.shape -# -# # bootstrap -# p_boots = [] -# for _ in range(self.bootstrap_trials): -# docs_idx = np.random.choice(num_docs, size=self.bootstra_range, replace=False) -# class_conditional_X = {i: X[docs_idx] for i, X in self.class_conditional_X.items()} -# Xboot = X[docs_idx] -# -# # bagging -# p_bags = [] -# for _ in range(self.bagging_trials): -# feat_idx = np.random.choice(num_feats, size=self.bagging_range, replace=False) -# class_conditional_Xbag = {i: X[:, feat_idx] for i, X in class_conditional_X.items()} -# Xbag = Xboot[:,feat_idx] -# p = self.std_constrained_linear_ls(Xbag, class_conditional_Xbag) -# p_bags.append(p) -# p_boots.append(np.mean(p_bags, axis=0)) -# -# p_mean = np.mean(p_boots, axis=0) -# p_std = np.std(p_bags, axis=0) -# -# return p_mean -# -# -# def std_constrained_linear_ls(self, X, class_cond_X: dict): -# pass +class EDx(_EnergyDistanceCore, BaseQuantifier): + """ + Energy Distance x (EDx), a covariate-space distribution-matching + quantifier based on energy distance. + + EDx is the classifier-free counterpart of :class:`quapy.method.aggregative.EDy`. + Instead of representing each class through posterior-probability vectors, it + represents each class by the cloud of raw feature vectors observed in the + training set and estimates the test prevalence vector by solving the same + energy-distance quadratic program directly in feature space. + + This implementation works for binary and multiclass single-label + quantification and relies on the optional ``quadprog`` dependency. The + current QuaPy adaptation shares its numerical core with EDy and keeps + credit to the original implementation available in + `quantificationlib `_. + + The formulation follows the same references as EDy, namely: + + * Alberto Castaño, Laura Morán-Fernández, Jaime Alonso, + Verónica Bolón-Canedo, Amparo Alonso-Betanzos, and Juan José del Coz. + *An analysis of quantification methods based on matching distributions*. + * Hideko Kawakubo, Marthinus Christoffel du Plessis, and Masashi Sugiyama + (2016). *Computationally efficient class-prior estimation under class + balance change using energy distance*. IEICE Transactions on Information + and Systems, 99(1):176-186. + + :param distance: distance used to compare feature vectors. Valid string + aliases are ``'manhattan'`` (default) and ``'euclidean'``; a custom + callable compatible with pairwise-distance signatures can also be used + :param n_jobs: number of parallel workers (default ``None``, meaning the + value is taken from the environment) + """ + + def __init__(self, distance: Union[str, Callable] = 'manhattan', n_jobs=None): + self.distance = distance + self.n_jobs = qp._get_njobs(n_jobs) + self.classes_ = None + self.n_features_in_ = None + self.train_distrib_ = None + self.train_n_cls_i_ = None + self.K_ = None + self.G_ = None + self.C_ = None + self.b_ = None + self.a_ = None + + def fit(self, X, y): + """Fit class-conditional feature-space distributions from training data.""" + self._check_ed_init_parameters() + labels = np.asarray(y) + self.classes_ = np.unique(labels) + self.n_features_in_ = X.shape[1] + train_distrib = [X[labels == class_] for class_ in self.classes_] + return self._fit_energy_model(train_distrib) + + def predict(self, X): + """Estimate class prevalences for a test sample of raw instances.""" + assert X.shape[1] == self.n_features_in_, ( + f'wrong shape; expected {self.n_features_in_}, found {X.shape[1]}' + ) + return self._predict_energy(X) + + +class ReadMe(BaseQuantifier, WithConfidenceABC): + """ + ReadMe is a non-aggregative quantification system proposed by + `Daniel Hopkins and Gary King, 2007. A method of automated nonparametric content analysis for + social science. American Journal of Political Science, 54(1):229–247. + `_. + The idea is to estimate `Q(Y=i)` directly from: + + :math:`Q(X)=\\sum_{i=1} Q(X|Y=i) Q(Y=i)` + + via least-squares regression, i.e., without incurring the cost of computing posterior probabilities. + However, this poses a very difficult representation in which the vector `Q(X)` and the matrix `Q(X|Y=i)` + can be of very high dimensions. In order to render the problem tracktable, ReadMe performs bagging in + the feature space. ReadMe also combines bagging with bootstrap in order to derive confidence intervals + around point estimations. + + We use the same default parameters as in the official + `R implementation `_. + + :param prob_model: str ('naive', or 'full'), selects the modality in which the probabilities `Q(X)` and + `Q(X|Y)` are to be modelled. Options include "full", which corresponds to the original formulation of + ReadMe, in which X is constrained to be a binary matrix (e.g., of term presence/absence) and in which + `Q(X)` and `Q(X|Y)` are modelled, respectively, as matrices of `(2^K, 1)` and `(2^K, n)` values, where + `K` is the number of columns in the data matrix (i.e., `bagging_range`), and `n` is the number of classes. + Of course, this approach is computationally prohibited for large `K`, so the authors advised against computing it + for matrices with `K>25` (although we recommend even smaller values of `K`). A much faster model is "naive", which + considers the `Q(X)` and `Q(X|Y)` be multinomial distributions under the `bag-of-words` perspective. In this + case, `bagging_range` can be set to much larger values. Default is "full" (i.e., original ReadMe behavior). + :param bootstrap_trials: int, number of bootstrap trials (default 300) + :param bagging_trials: int, number of bagging trials (default 300) + :param bagging_range: int, number of features to keep for each bagging trial (default 15) + :param confidence_level: float, a value in (0,1) reflecting the desired confidence level (default 0.95) + :param region: str in 'intervals', 'ellipse', 'ellipse-clr'; indicates the preferred method for + defining the confidence region (see :class:`WithConfidenceABC`) + :param bonferroni: bool (default False), whether to apply Bonferroni correction when + `region='intervals'`. This parameter has no effect for ellipse-based regions. + :param random_state: int or None, allows replicability (default None) + :param verbose: bool, whether to display information during the process (default False) + """ + + MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION = 25 + PROBABILISTIC_MODELS = ["naive", "full"] + + def __init__(self, + prob_model="full", + bootstrap_trials=300, + bagging_trials=300, + bagging_range=15, + confidence_level=0.95, + region='intervals', + bonferroni=False, + random_state=None, + verbose=False): + assert prob_model in ReadMe.PROBABILISTIC_MODELS, \ + f'unknown {prob_model=}, valid ones are {ReadMe.PROBABILISTIC_MODELS=}' + self.prob_model = prob_model + self.bootstrap_trials = bootstrap_trials + self.bagging_trials = bagging_trials + self.bagging_range = bagging_range + self.confidence_level = confidence_level + self.region = region + self.bonferroni = bonferroni + self.random_state = random_state + self.verbose = verbose + + def fit(self, X, y): + self._check_matrix(X) + + self.rng = np.random.default_rng(self.random_state) + self.classes_ = np.unique(y) + + Xsize = X.shape[0] + + # Bootstrap loop + self.Xboots, self.yboots = [], [] + for _ in range(self.bootstrap_trials): + idx = self.rng.choice(Xsize, size=Xsize, replace=True) + self.Xboots.append(X[idx]) + self.yboots.append(y[idx]) + + return self + + def predict_conf(self, X, confidence_level=None) -> (np.ndarray, ConfidenceRegionABC): + self._check_matrix(X) + if confidence_level is None: + confidence_level = self.confidence_level + + n_features = X.shape[1] + boots_prevalences = [] + for Xboots, yboots in tqdm( + zip(self.Xboots, self.yboots), + desc='bootstrap predictions', total=self.bootstrap_trials, disable=not self.verbose + ): + bagging_estimates = [] + for _ in range(self.bagging_trials): + feat_idx = self.rng.choice(n_features, size=self.bagging_range, replace=False) + Xboots_bagging = Xboots[:, feat_idx] + X_boots_bagging = X[:, feat_idx] + bagging_prev = self._quantify_iteration(Xboots_bagging, yboots, X_boots_bagging) + bagging_estimates.append(bagging_prev) + + boots_prevalences.append(np.mean(bagging_estimates, axis=0)) + + conf = WithConfidenceABC.construct_region(boots_prevalences, confidence_level, method=self.region, bonferroni=self.bonferroni) + prev_estim = conf.point_estimate() + + return prev_estim, conf + + def predict(self, X): + prev_estim, _ = self.predict_conf(X) + return prev_estim + + def _quantify_iteration(self, Xtr, ytr, Xte): + """Single ReadMe estimate.""" + PX_given_Y = np.asarray([self._compute_P(Xtr[ytr == c]) for i,c in enumerate(self.classes_)]) + PX = self._compute_P(Xte) + + res = lsq_linear(A=PX_given_Y.T, b=PX, bounds=(0, 1)) + pY = np.maximum(res.x, 0) + return pY / pY.sum() + + def _check_matrix(self, X): + """the "full" model requires estimating empirical distributions; due to the high computational cost, + this function is only made available for binary matrices""" + if self.prob_model == 'full' and not self._is_binary_matrix(X): + raise ValueError('the empirical distribution can only be computed efficiently on binary matrices') + + def _is_binary_matrix(self, X): + data = X.data if sparse.issparse(X) else X + return np.all((data == 0) | (data == 1)) + + def _compute_P(self, X): + if self.prob_model == 'naive': + return self._multinomial_distribution(X) + elif self.prob_model == 'full': + return self._empirical_distribution(X) + else: + raise ValueError(f'unknown {self.prob_model}; valid ones are {ReadMe.PROBABILISTIC_MODELS=}') + + def _empirical_distribution(self, X): + + if X.shape[1] > self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION: + raise ValueError(f'the empirical distribution can only be computed efficiently for dimensions ' + f'less or equal than {self.MAX_FEATURES_FOR_EMPIRICAL_ESTIMATION}') + + # we first convert every binary row (e.g., 0 0 1 0 1) into the equivalent number (e.g., 5); + # this will speed up subsequent comparisons a lot + K = X.shape[1] + binary_powers = 1 << np.arange(K-1, -1, -1) # (2^K, ..., 32, 16, 8, 4, 2, 1) + X_as_binary_numbers = X @ binary_powers # e.g., [0 0 1 0 1] @ [16, 8, 4, 2, 1] = 5 + + # count occurrences and compute probs + counts = np.bincount(X_as_binary_numbers, minlength=2 ** K).astype(float) + probs = counts / counts.sum() + + return probs + + def _multinomial_distribution(self, X): + PX = np.asarray(X.sum(axis=0)) + PX = normalize(PX, norm='l1', axis=1) + return PX.ravel() def _get_features_range(X): @@ -211,4 +402,8 @@ def _get_features_range(X): # aliases #--------------------------------------------------------------- -DistributionMatchingX = DMx \ No newline at end of file + +HDx = DMx.HDx +DistributionMatchingX = DMx +EnergyDistanceX = EDx +HellingerDistanceX = HDx \ No newline at end of file diff --git a/quapy/method/stan/pq.stan b/quapy/method/stan/pq.stan new file mode 100644 index 0000000..a5bc52a --- /dev/null +++ b/quapy/method/stan/pq.stan @@ -0,0 +1,39 @@ +data { + int n_bucket; + array[n_bucket] int train_pos; + array[n_bucket] int train_neg; + array[n_bucket] int test; + int posterior; +} + +transformed data{ + row_vector[n_bucket] train_pos_rv; + row_vector[n_bucket] train_neg_rv; + row_vector[n_bucket] test_rv; + real n_test; + + train_pos_rv = to_row_vector( train_pos ); + train_neg_rv = to_row_vector( train_neg ); + test_rv = to_row_vector( test ); + n_test = sum( test ); +} + +parameters { + simplex[n_bucket] p_neg; + simplex[n_bucket] p_pos; + real prev_prior; +} + +model { + if( posterior ) { + target += train_neg_rv * log( p_neg ); + target += train_pos_rv * log( p_pos ); + target += test_rv * log( p_neg * ( 1 - prev_prior) + p_pos * prev_prior ); + } +} + +generated quantities { + real prev; + prev = sum( binomial_rng(test, 1 / ( 1 + (p_neg./p_pos) *(1-prev_prior)/prev_prior ) ) ) / n_test; +} + diff --git a/quapy/model_selection.py b/quapy/model_selection.py index 0937fa8..c6af0c1 100644 --- a/quapy/model_selection.py +++ b/quapy/model_selection.py @@ -1,4 +1,5 @@ import itertools +import logging import signal from copy import deepcopy from enum import Enum @@ -91,7 +92,7 @@ class GridSearchQ(BaseQuantifier): def _sout(self, msg): if self.verbose: - print(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}') + logging.getLogger(__name__).info(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}') def __check_error_measure(self, error): if error in qp.error.QUANTIFICATION_ERROR: diff --git a/quapy/plot.py b/quapy/plot.py index 5c5e4b4..9d76aa9 100644 --- a/quapy/plot.py +++ b/quapy/plot.py @@ -1,12 +1,15 @@ from collections import defaultdict -import matplotlib.pyplot as plt -from matplotlib.pyplot import get_cmap -import numpy as np -from matplotlib import cm -from scipy.stats import ttest_ind_from_stats -from matplotlib.ticker import ScalarFormatter import math +from matplotlib import cm +import matplotlib.colors as mcolors +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap, ListedColormap +from matplotlib.pyplot import get_cmap +from matplotlib.ticker import ScalarFormatter +import numpy as np +from scipy.stats import ttest_ind_from_stats + import quapy as qp plt.rcParams['figure.figsize'] = [10, 6] @@ -480,7 +483,6 @@ def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs best_bucket_methods.append(method_order[method_index]) best_methods.append(best_bucket_methods) salient_methods.update(best_bucket_methods) - print(best_bucket_methods) if binning=='isomerous': fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods))) @@ -560,6 +562,123 @@ def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs return fig, ax +def plot_simplex( + point_layers=None, + region_layers=None, + density_function=None, + density_color='#1f77b4', + density_alpha=1.0, + resolution=400, + class_names=None, + title=None, + show_legend=True, + legend_loc='lower center', + legend_bbox_to_anchor=(0.5, -0.08), + legend_ncol=2, + figsize=(6.8, 6.2), + class_name_fontsize=10, + title_fontsize=11, + legend_fontsize=9, + ax=None, + savepath=None): + """ + Plots data on the ternary simplex for three-class quantification problems. + + This utility is convenient for visualising prevalence vectors, posterior triplets, + confidence regions, or any other points that lie on the 2-dimensional probability + simplex. The plot can combine three optional layer types: + + * `point_layers`: scatter layers for one or more prevalence clouds or reference points + * `region_layers`: shaded regions defined by callables on prevalence vectors + * `density_function`: a scalar function evaluated on the simplex and rendered as a heatmap + + Each entry in `point_layers` is a dictionary with a mandatory `points` field + containing an array-like of shape `(n_points, 3)` or `(3,)`. Optional fields are + `label` for the legend and `style` for matplotlib scatter keyword arguments. + + Each entry in `region_layers` is a dictionary with a mandatory `fn` field containing + a callable that receives prevalence vectors and returns region-membership scores. + Optional fields are `label`, `color`, and `alpha`. + + :param point_layers: optional list of point-layer dictionaries + :param region_layers: optional list of region-layer dictionaries + :param density_function: optional callable receiving prevalence vectors and returning + scalar values + :param density_color: color used for the density heatmap + :param density_alpha: opacity for the density heatmap + :param resolution: number of grid steps per axis used for rendering regions and densities + :param class_names: optional list or tuple with the three class names + :param title: optional plot title + :param show_legend: whether to display the legend + :param legend_loc: location string passed to matplotlib for the legend + :param legend_bbox_to_anchor: optional legend anchor box + :param legend_ncol: number of legend columns + :param figsize: figure size used when `ax` is not provided + :param class_name_fontsize: fontsize used for simplex vertex labels + :param title_fontsize: fontsize used for the optional title + :param legend_fontsize: fontsize used for the legend + :param ax: optional matplotlib axes object; if not provided, a new figure is created + :param savepath: path where to save the plot; if not indicated, the plot is shown when + `ax` is not provided + :return: returns `(fig, ax)` matplotlib objects for eventual customisation + """ + if class_names is None: + class_names = ('Y=1', 'Y=2', 'Y=3') + if len(class_names) != 3: + raise ValueError(f'expected exactly 3 class names, got {len(class_names)}') + + own_figure = ax is None + if own_figure: + fig, ax = plt.subplots(figsize=figsize) + else: + fig = ax.figure + + if density_function is not None: + _plot_simplex_density(ax, density_function, resolution, density_color, density_alpha) + + if region_layers: + _plot_simplex_regions(ax, region_layers, resolution) + + if point_layers: + _plot_simplex_points(ax, point_layers) + + simplex_ymax = np.sqrt(3) / 2 + triangle = np.array([ + [0.0, 0.0], + [1.0, 0.0], + [0.5, simplex_ymax], + [0.0, 0.0], + ]) + ax.plot(triangle[:, 0], triangle[:, 1], color='black') + + ax.text(-0.05, -0.05, class_names[0], ha='right', va='top', fontsize=class_name_fontsize) + ax.text(1.05, -0.05, class_names[1], ha='left', va='top', fontsize=class_name_fontsize) + ax.text(0.5, simplex_ymax + 0.05, class_names[2], ha='center', va='bottom', fontsize=class_name_fontsize) + + if title is not None: + ax.set_title(title, fontsize=title_fontsize) + + ax.set_aspect('equal') + ax.set_xlim(-0.1, 1.1) + ax.set_ylim(-0.1, simplex_ymax + 0.1) + ax.axis('off') + + if show_legend: + _, labels = ax.get_legend_handles_labels() + if labels: + ax.legend(loc=legend_loc, bbox_to_anchor=legend_bbox_to_anchor, ncol=legend_ncol, fontsize=legend_fontsize, frameon=False) + + fig.tight_layout(pad=0.6) + + if savepath is not None: + qp.util.create_parent_dir(savepath) + fig.savefig(savepath, bbox_inches='tight') + elif own_figure: + plt.show() + + return fig, ax + + def _merge(method_names, true_prevs, estim_prevs): ndims = true_prevs[0].shape[1] data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))}) @@ -612,6 +731,94 @@ def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error return data +def _simplex_to_cartesian(prevalences): + prevalences = np.asarray(prevalences, dtype=float) + prevalences = np.atleast_2d(prevalences) + if prevalences.shape[1] != 3: + raise ValueError(f'plot_simplex expects prevalence vectors of shape (_, 3); found {prevalences.shape}') + x = prevalences[:, 1] + 0.5 * prevalences[:, 2] + y = prevalences[:, 2] * (np.sqrt(3) / 2) + return x, y + + +def _barycentric_from_xy(x, y): + p3 = 2 * y / np.sqrt(3) + p2 = x - 0.5 * p3 + p1 = 1 - p2 - p3 + return np.stack([p1, p2, p3], axis=-1) + + +def _simplex_mesh(resolution): + simplex_ymax = np.sqrt(3) / 2 + xs = np.linspace(0, 1, resolution) + ys = np.linspace(0, simplex_ymax, resolution) + grid_x, grid_y = np.meshgrid(xs, ys) + pts_bary = _barycentric_from_xy(grid_x, grid_y) + mask = np.all(pts_bary >= 0, axis=-1) + return xs, ys, pts_bary, mask + + +def _evaluate_simplex_function(function, points): + points = np.asarray(points, dtype=float) + try: + values = np.asarray(function(points), dtype=float) + if values.shape == (points.shape[0],): + return values + if values.shape == points.shape[:-1]: + return values.reshape(-1) + except Exception: + pass + return np.asarray([function(point) for point in points], dtype=float) + + +def _region_colormap(color='blue', alpha=0.35): + return ListedColormap([ + (1.0, 1.0, 1.0, 0.0), + (*mcolors.to_rgb(color), alpha), + ]) + + +def _plot_simplex_points(ax, point_layers): + for layer in point_layers: + points = np.asarray(layer['points'], dtype=float) + style = {'s': 25, 'alpha': 0.8} + style.update(layer.get('style', {})) + ax.scatter(*_simplex_to_cartesian(points), label=layer.get('label'), **style) + + +def _plot_simplex_regions(ax, region_layers, resolution): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + + for layer in region_layers: + mask = np.zeros(simplex_mask.shape, dtype=float) + values = _evaluate_simplex_function(layer['fn'], valid_points) + mask[simplex_mask] = values + ax.pcolormesh( + xs, + ys, + mask, + shading='auto', + cmap=_region_colormap(layer.get('color', 'blue'), layer.get('alpha', 0.35)), + ) + if layer.get('label') is not None: + ax.scatter([], [], color=layer.get('color', 'blue'), alpha=layer.get('alpha', 0.35), label=layer['label']) + + +def _plot_simplex_density(ax, density_function, resolution, color, alpha): + xs, ys, pts_bary, simplex_mask = _simplex_mesh(resolution) + valid_points = pts_bary[simplex_mask] + density = np.full(simplex_mask.shape, np.nan, dtype=float) + values = _evaluate_simplex_function(density_function, valid_points) + min_v, max_v = np.min(values), np.max(values) + if max_v > min_v: + values = (values - min_v) / (max_v - min_v) + density[simplex_mask] = values + + cmap = LinearSegmentedColormap.from_list('simplex_density', ['white', color]) + ax.pcolormesh(xs, ys, density, shading='auto', cmap=cmap, alpha=alpha) + + def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None): posteriors = prob_classifier.predict_proba(X) assert posteriors.ndim==2, 'calibration plot only works for binary problems' @@ -619,7 +826,6 @@ def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None): pred_y = posteriors>=0.5 bins = np.linspace(0, 1, nbins + 1) binned_values = np.digitize(posteriors, bins, right=False) - print(np.unique(binned_values)) correct = pred_y == y bin_centers = (bins[:-1] + bins[1:]) / 2 bins_names = np.arange(nbins) diff --git a/quapy/protocol.py b/quapy/protocol.py index 9a7e5c4..4a7cb71 100644 --- a/quapy/protocol.py +++ b/quapy/protocol.py @@ -8,8 +8,8 @@ from contextlib import ExitStack from abc import ABCMeta, abstractmethod from quapy.data import LabelledCollection import quapy.functional as F -from os.path import exists -from glob import glob +from collections.abc import Iterable +from numbers import Number class AbstractProtocol(metaclass=ABCMeta): @@ -171,7 +171,7 @@ class AbstractStochasticSeededProtocol(AbstractProtocol): return sample -class OnLabelledCollectionProtocol: +class OnLabelledCollectionProtocol(AbstractStochasticSeededProtocol): """ Protocols that generate samples from a :class:`qp.data.LabelledCollection` object. """ @@ -229,8 +229,17 @@ class OnLabelledCollectionProtocol: elif return_type=='index': return lambda lc,params:params + def sample(self, index): + """ + Realizes the sample given the index of the instances. -class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): + :param index: indexes of the instances to select + :return: an instance of :class:`qp.data.LabelledCollection` + """ + return self.data.sampling_from_index(index) + + +class APP(OnLabelledCollectionProtocol): """ Implementation of the artificial prevalence protocol (APP). The APP consists of exploring a grid of prevalence values containing `n_prevalences` points (e.g., @@ -311,15 +320,6 @@ class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): - """ - Realizes the sample given the index of the instances. - - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` - """ - return self.data.sampling_from_index(index) - def total(self): """ Returns the number of samples that will be generated @@ -329,7 +329,7 @@ class APP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): return F.num_prevalence_combinations(self.n_prevalences, self.data.n_classes, self.repeats) -class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): +class NPP(OnLabelledCollectionProtocol): """ A generator of samples that implements the natural prevalence protocol (NPP). The NPP consists of drawing samples uniformly at random, therefore approximately preserving the natural prevalence of the collection. @@ -365,15 +365,6 @@ class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): - """ - Realizes the sample given the index of the instances. - - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` - """ - return self.data.sampling_from_index(index) - def total(self): """ Returns the number of samples that will be generated (equals to "repeats") @@ -383,7 +374,7 @@ class NPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): return self.repeats -class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): +class UPP(OnLabelledCollectionProtocol): """ A variant of :class:`APP` that, instead of using a grid of equidistant prevalence values, relies on the Kraemer algorithm for sampling unit (k-1)-simplex uniformly at random, with @@ -423,14 +414,63 @@ class UPP(AbstractStochasticSeededProtocol, OnLabelledCollectionProtocol): indexes.append(index) return indexes - def sample(self, index): + def total(self): """ - Realizes the sample given the index of the instances. + Returns the number of samples that will be generated (equals to "repeats") - :param index: indexes of the instances to select - :return: an instance of :class:`qp.data.LabelledCollection` + :return: int """ - return self.data.sampling_from_index(index) + return self.repeats + + +class DirichletProtocol(OnLabelledCollectionProtocol): + """ + A protocol that establishes a prior Dirichlet distribution for the prevalence of the samples. + Note that providing an all-ones vector of Dirichlet parameters is equivalent to invoking the + APP protocol (although each protocol will generate a different series of samples given a + fixed seed, since the implementation is different). + + :param data: a `LabelledCollection` from which the samples will be drawn + :param alpha: an array-like of shape (n_classes,) with the parameters of the Dirichlet distribution + :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from + qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised. + :param repeats: the number of samples to generate. Default is 100. + :param random_state: allows replicating samples across runs (default 0, meaning that the sequence of samples + will be the same every time the protocol is called) + :param return_type: set to "sample_prev" (default) to get the pairs of (sample, prevalence) at each iteration, or + to "labelled_collection" to get instead instances of LabelledCollection + """ + + def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0, + return_type='sample_prev'): + n_classes = data.n_classes + if isinstance(alpha, str) and alpha == 'uniform': + self.alpha = np.ones(n_classes, dtype=float) + elif isinstance(alpha, Number): + self.alpha = np.full(n_classes, float(alpha), dtype=float) + else: + self.alpha = np.asarray(alpha, dtype=float) + if self.alpha.ndim != 1 or len(self.alpha) != n_classes: + raise ValueError( + f'wrong shape for alpha; expected {n_classes} values, found shape {self.alpha.shape}' + ) + + super(DirichletProtocol, self).__init__(random_state) + self.data = data + self.sample_size = qp._get_sample_size(sample_size) + self.repeats = repeats + self.random_state = random_state + self.collator = OnLabelledCollectionProtocol.get_collator(return_type) + + def samples_parameters(self): + """ + Return all the necessary parameters to replicate the samples. + + :return: a list of indexes that realize the sampling + """ + prevs = np.random.dirichlet(self.alpha, size=self.repeats) + indexes = [self.data.sampling_index(self.sample_size, *prevs_i) for prevs_i in prevs] + return indexes def total(self): """ @@ -450,7 +490,7 @@ class DomainMixer(AbstractStochasticSeededProtocol): :param sample_size: integer, the number of instances in each sample; if None (default) then it is taken from qp.environ["SAMPLE_SIZE"]. If this is not set, a ValueError exception is raised. :param repeats: int, number of samples to draw for every mixture rate - :param prevalence: the prevalence to preserv along the mixtures. If specified, should be an array containing + :param prevalence: the prevalence to preserve along the mixtures. If specified, should be an array containing one prevalence value (positive float) for each class and summing up to one. If not specified, the prevalence will be taken from the domain A (default). :param mixture_points: an integer indicating the number of points to take from a linear scale (e.g., 21 will diff --git a/quapy/tests/__init__.py b/quapy/tests/__init__.py index e69de29..374e4b0 100644 --- a/quapy/tests/__init__.py +++ b/quapy/tests/__init__.py @@ -0,0 +1,10 @@ +""" +Fast unit tests live in ``test_*.py`` and are intended to run without network +access or large external resources. + +Slow integration tests that download datasets or depend on optional stacks live +in ``integration_*.py`` and are meant to be run explicitly, e.g.: + + python -m unittest quapy.tests.integration_datasets + python -m unittest quapy.tests.integration_methods +""" diff --git a/quapy/tests/_synthetic.py b/quapy/tests/_synthetic.py new file mode 100644 index 0000000..84be4f2 --- /dev/null +++ b/quapy/tests/_synthetic.py @@ -0,0 +1,48 @@ +import numpy as np +from sklearn.datasets import make_classification + +from quapy.data import LabelledCollection +from quapy.data.base import Dataset + + +def make_labelled_collection( + n_samples=200, + n_features=12, + n_classes=2, + class_sep=1.5, + random_state=0, +): + n_informative = min(n_features, max(4, n_classes * 2)) + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_informative, + n_redundant=0, + n_repeated=0, + n_classes=n_classes, + n_clusters_per_class=1, + class_sep=class_sep, + random_state=random_state, + ) + classes = np.arange(n_classes) + return LabelledCollection(X, y, classes=classes) + + +def make_dataset( + n_train=150, + n_test=80, + n_features=12, + n_classes=2, + class_sep=1.5, + random_state=0, + name='synthetic', +): + data = make_labelled_collection( + n_samples=n_train + n_test, + n_features=n_features, + n_classes=n_classes, + class_sep=class_sep, + random_state=random_state, + ) + training, test = data.split_stratified(train_prop=n_train / (n_train + n_test), random_state=random_state) + return Dataset(training, test, name=name) diff --git a/quapy/tests/test_datasets.py b/quapy/tests/integration_datasets.py similarity index 75% rename from quapy/tests/test_datasets.py rename to quapy/tests/integration_datasets.py index de5f61a..1e8445b 100644 --- a/quapy/tests/test_datasets.py +++ b/quapy/tests/integration_datasets.py @@ -1,3 +1,10 @@ +""" +Integration tests for dataset fetchers and large external resources. + +This module is intentionally excluded from default ``unittest`` discovery by +using an ``integration_*.py`` filename. +""" + import os import unittest @@ -5,11 +12,11 @@ from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression import quapy.functional as F -from quapy.method.aggregative import PCC from quapy.data.datasets import * +from quapy.method.aggregative import PCC -class TestDatasets(unittest.TestCase): +class IntegrationDatasetsTest(unittest.TestCase): def new_quantifier(self): return PCC(LogisticRegression(C=0.001, max_iter=100)) @@ -17,13 +24,11 @@ class TestDatasets(unittest.TestCase): def _check_dataset(self, dataset): train, test = dataset.reduce().train_test q = self.new_quantifier() - print(f'testing method {q} in {dataset.name}...', end='') - if len(train)>500: + if len(train) > 500: train = train.sampling(500) q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.instances) self.assertTrue(F.check_prevalence_vector(estim_prevalences)) - print(f'[done]') def _check_samples(self, gen, q, max_samples_test=5, vectorizer=None): for X, p in gen(): @@ -37,54 +42,37 @@ class TestDatasets(unittest.TestCase): def test_reviews(self): for dataset_name in REVIEWS_SENTIMENT_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_reviews(dataset_name, tfidf=True, min_df=10) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_twitter(self): - # all the datasets are contained in the same resource; if the first one - # works, there is no need to test for the rest for dataset_name in TWITTER_SENTIMENT_DATASETS_TEST[:1]: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_twitter(dataset_name, min_df=10) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_UCIBinaryDataset(self): for dataset_name in UCI_BINARY_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_UCIBinaryDataset(dataset_name) - dataset.stats() dataset.reduce() - print(f'[done]') self._check_dataset(dataset) def test_UCIMultiDataset(self): for dataset_name in UCI_MULTICLASS_DATASETS: - print(f'loading dataset {dataset_name}...', end='') dataset = fetch_UCIMulticlassDataset(dataset_name) - dataset.stats() n_classes = dataset.n_classes uniform_prev = F.uniform_prevalence(n_classes) dataset.training = dataset.training.sampling(100, *uniform_prev) dataset.test = dataset.test.sampling(100, *uniform_prev) - print(f'[done]') self._check_dataset(dataset) def test_lequa2022(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_lequa2022 because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return for dataset_name in LEQUA2022_VECTOR_TASKS: - print(f'LeQu2022: loading dataset {dataset_name}...', end='') train, gen_val, gen_test = fetch_lequa2022(dataset_name) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() @@ -93,9 +81,7 @@ class TestDatasets(unittest.TestCase): self._check_samples(gen_test, q, max_samples_test=5) for dataset_name in LEQUA2022_TEXT_TASKS: - print(f'LeQu2022: loading dataset {dataset_name}...', end='') train, gen_val, gen_test = fetch_lequa2022(dataset_name) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) tfidf = TfidfVectorizer() @@ -107,13 +93,10 @@ class TestDatasets(unittest.TestCase): def test_lequa2024(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_lequa2024 because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return for task in LEQUA2024_TASKS: - print(f'LeQu2024: loading task {task}...', end='') train, gen_val, gen_test = fetch_lequa2024(task, merge_T3=True) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() @@ -121,16 +104,12 @@ class TestDatasets(unittest.TestCase): self._check_samples(gen_val, q, max_samples_test=5) self._check_samples(gen_test, q, max_samples_test=5) - def test_IFCB(self): if os.environ.get('QUAPY_TESTS_OMIT_LARGE_DATASETS'): - print("omitting test_IFCB because QUAPY_TESTS_OMIT_LARGE_DATASETS is set") return - print(f'loading dataset IFCB.') for mod_sel in [False, True]: train, gen = fetch_IFCB(single_sample_train=True, for_model_selection=mod_sel) - train.stats() n_classes = train.n_classes train = train.sampling(100, *F.uniform_prevalence(n_classes)) q = self.new_quantifier() diff --git a/quapy/tests/integration_methods.py b/quapy/tests/integration_methods.py new file mode 100644 index 0000000..180d80e --- /dev/null +++ b/quapy/tests/integration_methods.py @@ -0,0 +1,42 @@ +""" +Integration tests for optional or resource-heavy method end-to-end checks. + +This module is intentionally excluded from default ``unittest`` discovery by +using an ``integration_*.py`` filename. +""" + +import unittest + +import quapy as qp +from quapy.functional import check_prevalence_vector + + +class IntegrationMethodsTest(unittest.TestCase): + + def test_quanet(self): + try: + import quapy.classification.neural + except ModuleNotFoundError: + print('the torch package is not installed; skipping integration test for QuaNet') + return + + qp.environ['SAMPLE_SIZE'] = 10 + + dataset = qp.datasets.fetch_reviews('kindle', pickle=True).reduce() + qp.data.preprocessing.index(dataset, min_df=5, inplace=True) + + from quapy.classification.neural import CNNnet + from quapy.classification.neural import NeuralClassifierTrainer + from quapy.method.meta import QuaNet + + cnn = CNNnet(dataset.vocabulary_size, dataset.n_classes) + learner = NeuralClassifierTrainer(cnn, device='cpu') + model = QuaNet(learner, device='cpu', n_epochs=2, tr_iter_per_poch=10, va_iter_per_poch=10, patience=2) + + model.fit(*dataset.training.Xy) + estim_prevalences = model.predict(dataset.test.instances) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_bayesian_utils.py b/quapy/tests/test_bayesian_utils.py new file mode 100644 index 0000000..8ac6058 --- /dev/null +++ b/quapy/tests/test_bayesian_utils.py @@ -0,0 +1,69 @@ +import unittest + +import numpy as np + +from quapy.method._bayesian import ( + _validate_temperature, + _resolve_dirichlet_prior, + kl_div, + js_div, + normalized, + lambda_inverse, + lambda_forward, +) + + +class TestBayesianUtils(unittest.TestCase): + """ + Smoke tests for the pure-numpy helper functions in method/_bayesian.py. These do not require the + optional jax/stan/numpyro dependencies (unlike BayesianKDEy/BayesianMAPLS themselves), so they can + run regardless of whether `quapy[bayes]` is installed. + """ + + def test_validate_temperature(self): + self.assertEqual(_validate_temperature(2), 2.0) + self.assertEqual(_validate_temperature(0.5), 0.5) + with self.assertRaises(ValueError): + _validate_temperature(0) + with self.assertRaises(ValueError): + _validate_temperature(-1) + with self.assertRaises(ValueError): + _validate_temperature('not-a-number') + + def test_resolve_dirichlet_prior(self): + np.testing.assert_array_equal(_resolve_dirichlet_prior('uniform', n_classes=3), np.ones(3)) + np.testing.assert_array_equal(_resolve_dirichlet_prior(2.5, n_classes=3), np.full(3, 2.5)) + np.testing.assert_array_equal(_resolve_dirichlet_prior([1, 2, 3], n_classes=3), np.array([1., 2., 3.])) + with self.assertRaises(ValueError): + _resolve_dirichlet_prior([1, 2], n_classes=3) # wrong shape + with self.assertRaises(ValueError): + _resolve_dirichlet_prior('unknown-prior', n_classes=3) + + def test_kl_div_and_js_div(self): + p = np.array([0.5, 0.5]) + self.assertAlmostEqual(kl_div(p, p), 0.0, places=6) + self.assertAlmostEqual(js_div(p, p), 0.0, places=6) + + q = np.array([0.9, 0.1]) + self.assertGreater(kl_div(p, q), 0.0) + self.assertGreater(js_div(p, q), 0.0) + # JS divergence is symmetric, unlike KL + self.assertAlmostEqual(js_div(p, q), js_div(q, p), places=6) + + def test_normalized(self): + # normalized() operates on a batch of row vectors, shape (n_samples, n_features) + a = np.array([[3.0, 4.0], [0.0, 0.0]]) + normed = normalized(a) + self.assertAlmostEqual(np.linalg.norm(normed[0]), 1.0, places=6) + # an all-zero row should not raise (guarded division), and stays all-zero + np.testing.assert_array_equal(normed[1], np.array([0.0, 0.0])) + + def test_lambda_inverse_forward_roundtrip(self): + dpq, lam = 0.3, 0.4 + gamma = lambda_inverse(dpq, lam) + recovered_lam = lambda_forward(dpq, gamma) + self.assertAlmostEqual(recovered_lam, lam, places=6) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_calibration.py b/quapy/tests/test_calibration.py new file mode 100644 index 0000000..4bdaae4 --- /dev/null +++ b/quapy/tests/test_calibration.py @@ -0,0 +1,35 @@ +import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + +from quapy.classification.calibration import NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration +from quapy.tests._synthetic import make_labelled_collection + + +class TestCalibration(unittest.TestCase): + + data = make_labelled_collection(n_samples=200, n_features=10, n_classes=3, random_state=23) + + def test_calibration_methods_fit_predict_proba(self): + X, y = self.data.Xy + for calib_cls in [NBVSCalibration, BCTSCalibration, TSCalibration, VSCalibration]: + model = calib_cls(LogisticRegression(max_iter=2000), val_split=5) + model.fit(X, y) + posteriors = model.predict_proba(X) + self.assertEqual(posteriors.shape, (len(y), self.data.n_classes)) + np.testing.assert_allclose(posteriors.sum(axis=1), 1.0, rtol=1e-5, + err_msg=f'{calib_cls.__name__} posteriors do not sum to 1') + predictions = model.predict(X) + self.assertEqual(len(predictions), len(y)) + + def test_calibration_with_float_val_split(self): + X, y = self.data.Xy + model = BCTSCalibration(LogisticRegression(max_iter=2000), val_split=0.3) + model.fit(X, y) + posteriors = model.predict_proba(X) + self.assertEqual(posteriors.shape, (len(y), self.data.n_classes)) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_confidence.py b/quapy/tests/test_confidence.py new file mode 100644 index 0000000..9197597 --- /dev/null +++ b/quapy/tests/test_confidence.py @@ -0,0 +1,89 @@ +import unittest + +import numpy as np +from sklearn.linear_model import LogisticRegression + +from quapy.method.aggregative import PACC +from quapy.method.confidence import ConfidenceIntervals, ConfidenceEllipseSimplex, AggregativeBootstrap, WithConfidenceABC +from quapy.tests._synthetic import make_dataset + + +def _dirichlet_samples(n_classes=3, n_samples=300, random_state=0): + rng = np.random.RandomState(random_state) + return rng.dirichlet(np.ones(n_classes) * 5, size=n_samples) + + +class TestConfidenceRegions(unittest.TestCase): + + def test_confidence_intervals_contain_own_mean(self): + samples = _dirichlet_samples() + region = ConfidenceIntervals(samples) + point_estimate = region.point_estimate() + self.assertEqual(region.coverage(point_estimate), 1.) + self.assertEqual(region.n_dim, 3) + + def test_confidence_ellipse_simplex_contains_own_mean(self): + samples = _dirichlet_samples() + region = ConfidenceEllipseSimplex(samples) + point_estimate = region.point_estimate() + self.assertIn(point_estimate, region) + + def test_construct_region_applies_bonferroni_only_to_intervals(self): + samples = _dirichlet_samples() + region_plain = WithConfidenceABC.construct_region(samples, confidence_level=0.9, method='intervals') + region_bonf = WithConfidenceABC.construct_region(samples, confidence_level=0.9, method='intervals', bonferroni=True) + ellipse = WithConfidenceABC.construct_region(samples, confidence_level=0.9, method='ellipse', bonferroni=True) + + self.assertAlmostEqual(region_plain.alpha, 0.1) + self.assertAlmostEqual(region_bonf.alpha, 0.1 / samples.shape[1]) + self.assertIsInstance(ellipse, ConfidenceEllipseSimplex) + + def test_simplex_portion_is_cached_and_consistent(self): + # regression test for the @lru_cache-on-bound-method memory leak fix: + # results must still be memoized per instance, and two instances must not share state + region1 = ConfidenceEllipseSimplex(_dirichlet_samples(random_state=1)) + region2 = ConfidenceEllipseSimplex(_dirichlet_samples(random_state=2)) + + p1_first = region1.simplex_portion() + p1_second = region1.simplex_portion() + self.assertEqual(p1_first, p1_second) + + p2 = region2.simplex_portion() + self.assertTrue(hasattr(region1, '_simplex_portion_cache')) + self.assertTrue(hasattr(region2, '_simplex_portion_cache')) + # each instance keeps its own cached value + self.assertEqual(region1._simplex_portion_cache, p1_first) + self.assertEqual(region2._simplex_portion_cache, p2) + + def test_aggregative_bootstrap_end_to_end(self): + dataset = make_dataset(n_train=150, n_test=50, n_classes=3, n_features=12, random_state=5) + learner = LogisticRegression(max_iter=2000) + learner.fit(*dataset.training.Xy) + quantifier = AggregativeBootstrap( + PACC(learner, fit_classifier=False), n_test_samples=50, confidence_level=0.9 + ) + quantifier.fit(*dataset.training.Xy) + point_estimate, region = quantifier.predict_conf(dataset.test.X) + self.assertEqual(len(point_estimate), 3) + self.assertEqual(region.coverage(point_estimate), 1.) + + def test_aggregative_bootstrap_exposes_bonferroni(self): + dataset = make_dataset(n_train=150, n_test=50, n_classes=3, n_features=12, random_state=7) + learner = LogisticRegression(max_iter=2000) + learner.fit(*dataset.training.Xy) + quantifier = AggregativeBootstrap( + PACC(learner, fit_classifier=False), + n_test_samples=20, + confidence_level=0.9, + region='intervals', + bonferroni=True, + random_state=0, + ) + quantifier.fit(*dataset.training.Xy) + _, region = quantifier.predict_conf(dataset.test.X) + + self.assertAlmostEqual(region.alpha, 0.1 / 3) + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_evaluation.py b/quapy/tests/test_evaluation.py index 05d661a..871b6ec 100644 --- a/quapy/tests/test_evaluation.py +++ b/quapy/tests/test_evaluation.py @@ -1,43 +1,41 @@ +import inspect import unittest - -import numpy as np - -import quapy as qp -from sklearn.linear_model import LogisticRegression from time import time +import numpy as np +from sklearn.linear_model import LogisticRegression + +import quapy as qp from quapy.error import QUANTIFICATION_ERROR_SINGLE_NAMES from quapy.method.aggregative import EMQ, PCC from quapy.method.base import BaseQuantifier +from quapy.tests._synthetic import make_dataset class EvalTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.data = make_dataset(n_train=140, n_test=90, n_classes=2, random_state=7, name='eval') + def test_eval_speedup(self): - """ - Checks whether the speed-up heuristics used by qp.evaluation work, i.e., actually save time - """ - - data = qp.datasets.fetch_reviews('hp', tfidf=True, min_df=10, pickle=True) - train, test = data.training, data.test - - protocol = qp.protocol.APP(test, sample_size=1000, n_prevalences=11, repeats=1, random_state=1) + train, test = self.data.training, self.data.test + protocol = qp.protocol.APP(test, sample_size=30, n_prevalences=5, repeats=1, random_state=1) class SlowLR(LogisticRegression): def predict_proba(self, X): - import time - time.sleep(1) + import time as _time + _time.sleep(0.05) return super().predict_proba(X) - emq = EMQ(SlowLR()).fit(*train.Xy) + emq = EMQ(SlowLR(max_iter=1000)).fit(*train.Xy) tinit = time() - score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', verbose=True, aggr_speedup='force') - tend_optim = time()-tinit - print(f'evaluation (with optimization) took {tend_optim}s [MAE={score:.4f}]') + score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', aggr_speedup='force') + tend_optim = time() - tinit + self.assertTrue(isinstance(score, float)) class NonAggregativeEMQ(BaseQuantifier): - def __init__(self, cls): self.emq = EMQ(cls) @@ -48,31 +46,32 @@ class EvalTestCase(unittest.TestCase): self.emq.fit(X, y) return self - emq = NonAggregativeEMQ(SlowLR()).fit(*train.Xy) + emq = NonAggregativeEMQ(SlowLR(max_iter=1000)).fit(*train.Xy) tinit = time() - score = qp.evaluation.evaluate(emq, protocol, error_metric='mae', verbose=True) + score = qp.evaluation.evaluate(emq, protocol, error_metric='mae') tend_no_optim = time() - tinit - print(f'evaluation (w/o optimization) took {tend_no_optim}s [MAE={score:.4f}]') - - self.assertEqual(tend_no_optim>(tend_optim/2), True) + self.assertTrue(isinstance(score, float)) + self.assertGreater(tend_no_optim, tend_optim) def test_evaluation_output(self): - """ - Checks the evaluation functions return correct types for different error_metrics - """ + train, test = self.data.training, self.data.test + qp.environ['SAMPLE_SIZE'] = 30 + protocol = qp.protocol.APP(test, sample_size=30, n_prevalences=5, repeats=1, random_state=0) + q = PCC(LogisticRegression(max_iter=1000)).fit(*train.Xy) - data = qp.datasets.fetch_reviews('hp', tfidf=True, min_df=10, pickle=True).reduce(n_train=100, n_test=100) - train, test = data.training, data.test + def supports_evaluation(err): + required = [ + p for p in inspect.signature(err).parameters.values() + if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.default is inspect._empty + ] + return len(required) <= 2 - qp.environ['SAMPLE_SIZE']=100 - - protocol = qp.protocol.APP(test, random_state=0) - - q = PCC(LogisticRegression()).fit(*train.Xy) - - single_errors = list(QUANTIFICATION_ERROR_SINGLE_NAMES) - averaged_errors = ['m'+e for e in single_errors] + single_errors = [ + e for e in QUANTIFICATION_ERROR_SINGLE_NAMES + if supports_evaluation(qp.error.from_name(e)) + ] + averaged_errors = ['m' + e for e in single_errors] single_errors = single_errors + [qp.error.from_name(e) for e in single_errors] averaged_errors = averaged_errors + [qp.error.from_name(e) for e in averaged_errors] for error_metric, averaged_error_metric in zip(single_errors, averaged_errors): @@ -81,7 +80,6 @@ class EvalTestCase(unittest.TestCase): scores = qp.evaluation.evaluate(q, protocol, error_metric=error_metric) self.assertTrue(isinstance(scores, np.ndarray)) - self.assertEqual(scores.mean(), score) diff --git a/quapy/tests/test_functional.py b/quapy/tests/test_functional.py new file mode 100644 index 0000000..97e2307 --- /dev/null +++ b/quapy/tests/test_functional.py @@ -0,0 +1,27 @@ +import unittest + +import numpy as np + +import quapy.functional as F + + +class TestFunctional(unittest.TestCase): + + def test_ternary_search_binary(self): + def loss(prev): + return (prev[1] - 0.37) ** 2 + + result = F.argmin_prevalence(loss, n_classes=2, method='ternary_search') + self.assertTrue(np.allclose(result.sum(), 1.0)) + self.assertAlmostEqual(result[1], 0.37, places=3) + + def test_ternary_search_multiclass_not_supported(self): + def loss(prev): + return np.sum((prev - np.array([0.2, 0.3, 0.5])) ** 2) + + with self.assertRaises(AssertionError): + F.argmin_prevalence(loss, n_classes=3, method='ternary_search') + + +if __name__ == '__main__': + unittest.main() diff --git a/quapy/tests/test_methods.py b/quapy/tests/test_methods.py index 3e4149e..e059a4d 100644 --- a/quapy/tests/test_methods.py +++ b/quapy/tests/test_methods.py @@ -1,130 +1,226 @@ import itertools +import inspect import unittest +import numpy as np + from sklearn.linear_model import LogisticRegression -import quapy as qp -from quapy.method.aggregative import ACC -from quapy.method.meta import Ensemble from quapy.method import AGGREGATIVE_METHODS, BINARY_METHODS, NON_AGGREGATIVE_METHODS +from quapy.method.non_aggregative import DMx, EDx, HDx +from quapy.method.aggregative import ACC, DMy, EDy, KDEyCS, RLLS +from quapy.method.meta import Ensemble from quapy.functional import check_prevalence_vector +from quapy.tests._synthetic import make_dataset +import quapy as qp -# a random selection of composed methods to test the qunfold integration -from quapy.method.composable import check_compatible_qunfold_version +OPTIONAL_AGGREGATIVE_METHODS = { + 'BayesianCC', + 'BayesianKDEy', + 'BayesianMAPLS', + 'PQ', + 'RLLS', + 'EDy', +} -from quapy.method.composable import ( - ComposableQuantifier, - LeastSquaresLoss, - HellingerSurrogateLoss, - ClassRepresentation, - HistogramRepresentation, - CVClassifier -) +OPTIONAL_NON_AGGREGATIVE_METHODS = { + 'EDx', +} -COMPOSABLE_METHODS = [ - ComposableQuantifier( # ACC - LeastSquaresLoss(), - ClassRepresentation(CVClassifier(LogisticRegression())) - ), - ComposableQuantifier( # HDy - HellingerSurrogateLoss(), - HistogramRepresentation( - 3, # 3 bins per class - preprocessor = ClassRepresentation(CVClassifier(LogisticRegression())) - ) - ), -] class TestMethods(unittest.TestCase): - tiny_dataset_multiclass = qp.datasets.fetch_UCIMulticlassDataset('academic-success').reduce(n_test=10) - tiny_dataset_binary = qp.datasets.fetch_UCIBinaryDataset('ionosphere').reduce(n_test=10) + tiny_dataset_multiclass = make_dataset( + n_train=140, n_test=40, n_classes=3, n_features=12, random_state=11, name='synthetic-multiclass' + ) + tiny_dataset_binary = make_dataset( + n_train=140, n_test=40, n_classes=2, n_features=12, random_state=13, name='synthetic-binary' + ) datasets = [tiny_dataset_binary, tiny_dataset_multiclass] def test_aggregative(self): for dataset in TestMethods.datasets: - learner = LogisticRegression() + learner = LogisticRegression(max_iter=2000) learner.fit(*dataset.training.Xy) for model in AGGREGATIVE_METHODS: + if model.__name__ in OPTIONAL_AGGREGATIVE_METHODS: + continue if not dataset.binary and model in BINARY_METHODS: - print(f'skipping the test of binary model {model.__name__} on multiclass dataset {dataset.name}') continue - q = model(learner, fit_classifier=False) - print('testing', q) + kwargs = {'fit_classifier': False} + if 'val_split' in inspect.signature(model.__init__).parameters: + kwargs['val_split'] = None + q = model(learner, **kwargs) q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) self.assertTrue(check_prevalence_vector(estim_prevalences)) def test_non_aggregative(self): for dataset in TestMethods.datasets: - for model in NON_AGGREGATIVE_METHODS: + if model.__name__ in OPTIONAL_NON_AGGREGATIVE_METHODS: + continue if not dataset.binary and model in BINARY_METHODS: - print(f'skipping the test of binary model {model.__name__} on multiclass dataset {dataset.name}') continue q = model() - print(f'testing {q} on dataset {dataset.name}') q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) self.assertTrue(check_prevalence_vector(estim_prevalences)) def test_ensembles(self): - qp.environ['SAMPLE_SIZE'] = 10 + qp.environ['SAMPLE_SIZE'] = 20 - base_quantifier = ACC(LogisticRegression()) + def policy_supported(policy): + if policy in {'ave', 'ptr', 'ds'}: + return True + err = qp.error.from_name(policy) + required = [ + p for p in inspect.signature(err).parameters.values() + if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) and p.default is inspect._empty + ] + return len(required) <= 2 + + base_quantifier = ACC(LogisticRegression(max_iter=2000)) for dataset, policy in itertools.product(TestMethods.datasets, Ensemble.VALID_POLICIES): + if not policy_supported(policy): + continue if not dataset.binary and policy == 'ds': - print(f'skipping the test of binary policy ds on non-binary dataset {dataset}') continue - print(f'testing {base_quantifier} on dataset {dataset.name} with {policy=}') - ensemble = Ensemble(quantifier=base_quantifier, size=3, policy=policy, n_jobs=-1) + ensemble = Ensemble(quantifier=base_quantifier, size=3, policy=policy, n_jobs=1) ensemble.fit(*dataset.training.Xy) estim_prevalences = ensemble.predict(dataset.test.instances) self.assertTrue(check_prevalence_vector(estim_prevalences)) - def test_quanet(self): + def test_composable(self): try: - import quapy.classification.neural - except ModuleNotFoundError: - print('the torch package is not installed; skipping unit test for QuaNet') + from quapy.method.composable import check_compatible_qunfold_version + from quapy.method.composable import ( + ComposableQuantifier, + LeastSquaresLoss, + HellingerSurrogateLoss, + ClassRepresentation, + HistogramRepresentation, + CVClassifier, + ) + except ImportError: return - qp.environ['SAMPLE_SIZE'] = 10 + composable_methods = [ + ComposableQuantifier( + LeastSquaresLoss(), + ClassRepresentation(CVClassifier(LogisticRegression())) + ), + ComposableQuantifier( + HellingerSurrogateLoss(), + HistogramRepresentation( + 3, + preprocessor=ClassRepresentation(CVClassifier(LogisticRegression())) + ) + ), + ] - # load the kindle dataset as text, and convert words to numerical indexes - dataset = qp.datasets.fetch_reviews('kindle', pickle=True).reduce() - qp.data.preprocessing.index(dataset, min_df=5, inplace=True) - - from quapy.classification.neural import CNNnet - cnn = CNNnet(dataset.vocabulary_size, dataset.n_classes) - - from quapy.classification.neural import NeuralClassifierTrainer - learner = NeuralClassifierTrainer(cnn, device='cpu') - - from quapy.method.meta import QuaNet - model = QuaNet(learner, device='cpu', n_epochs=2, tr_iter_per_poch=10, va_iter_per_poch=10, patience=2) - - model.fit(*dataset.training.Xy) - estim_prevalences = model.predict(dataset.test.instances) - self.assertTrue(check_prevalence_vector(estim_prevalences)) - - def test_composable(self): if check_compatible_qunfold_version(): for dataset in TestMethods.datasets: - for q in COMPOSABLE_METHODS: - print('testing', q) + for q in composable_methods: q.fit(*dataset.training.Xy) estim_prevalences = q.predict(dataset.test.X) - print(estim_prevalences) self.assertTrue(check_prevalence_vector(estim_prevalences)) else: from quapy.method.composable import __old_version_message print(__old_version_message) + def test_rlls(self): + try: + import cvxpy # noqa: F401 + except ImportError: + return + + dataset = TestMethods.tiny_dataset_multiclass + q = RLLS(LogisticRegression(max_iter=2000), val_split=3) + q.fit(*dataset.training.Xy) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + + + def test_edy(self): + try: + import quadprog # noqa: F401 + except ImportError: + return + + dataset = TestMethods.tiny_dataset_multiclass + q = EDy(LogisticRegression(max_iter=2000), val_split=3) + q.fit(*dataset.training.Xy) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + + + def test_edx(self): + try: + import quadprog # noqa: F401 + except ImportError: + return + + dataset = TestMethods.tiny_dataset_multiclass + q = EDx() + q.fit(*dataset.training.Xy) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + + + def test_dmy_noncanonical_labels(self): + dataset = TestMethods.tiny_dataset_multiclass + label_names = np.asarray(['class-a', 'class-c', 'class-z']) + y_train = label_names[dataset.training.y] + y_test = label_names[dataset.test.y] + + q = DMy(LogisticRegression(max_iter=2000), val_split=3) + q.fit(dataset.training.X, y_train) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + self.assertEqual(len(estim_prevalences), len(np.unique(y_test))) + + + def test_dmx_noncanonical_labels(self): + dataset = TestMethods.tiny_dataset_multiclass + label_names = np.asarray(['class-a', 'class-c', 'class-z']) + y_train = label_names[dataset.training.y] + + q = DMx() + q.fit(dataset.training.X, y_train) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + self.assertEqual(len(estim_prevalences), len(np.unique(y_train))) + + def test_kdeycs_noncanonical_labels(self): + dataset = TestMethods.tiny_dataset_multiclass + label_names = np.asarray(['class-a', 'class-c', 'class-z']) + y_train = label_names[dataset.training.y] + + q = KDEyCS(LogisticRegression(max_iter=2000), val_split=3) + q.fit(dataset.training.X, y_train) + estim_prevalences = q.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(estim_prevalences)) + self.assertEqual(len(estim_prevalences), len(np.unique(y_train))) + + + def test_historical_distribution_matching_presets(self): + dataset = TestMethods.tiny_dataset_binary + + hdy = DMy.HDy(LogisticRegression(max_iter=2000), val_split=3) + hdy.fit(*dataset.training.Xy) + prev_hdy = hdy.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(prev_hdy)) + + hdx = HDx() + hdx.fit(*dataset.training.Xy) + prev_hdx = hdx.predict(dataset.test.X) + self.assertTrue(check_prevalence_vector(prev_hdx)) + if __name__ == '__main__': unittest.main() diff --git a/quapy/tests/test_modsel.py b/quapy/tests/test_modsel.py index 6423b4e..0f3c8f6 100644 --- a/quapy/tests/test_modsel.py +++ b/quapy/tests/test_modsel.py @@ -1,104 +1,87 @@ +import time import unittest import numpy as np from sklearn.linear_model import LogisticRegression -import quapy as qp from quapy.method.aggregative import PACC from quapy.model_selection import GridSearchQ from quapy.protocol import APP -import time +from quapy.tests._synthetic import make_dataset class ModselTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + data = make_dataset( + n_train=220, + n_test=120, + n_classes=2, + n_features=16, + class_sep=1.8, + random_state=1, + name='modsel', + ) + cls.training, cls.validation = data.training.split_stratified(0.7, random_state=1) + def test_modsel(self): """ - Checks whether a model selection exploration takes a good hyperparameter + Checks whether a model selection exploration picks the better hyperparameter. """ - q = PACC(LogisticRegression(random_state=1, max_iter=5000)) - - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=10).reduce(random_state=1) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': [0.000001, 10.]} - app = APP(validation, sample_size=100, random_state=1) + param_grid = {'classifier__C': [0.000001, 10.0]} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) q = GridSearchQ( - q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, verbose=True, n_jobs=-1 - ).fit(*training.Xy) - print('best params', q.best_params_) - print('best score', q.best_score_) + q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, verbose=False, n_jobs=-1 + ).fit(*self.training.Xy) self.assertEqual(q.best_params_['classifier__C'], 10.0) self.assertEqual(q.best_model().get_params()['classifier__C'], 10.0) def test_modsel_parallel(self): """ - Checks whether a parallelized model selection actually is faster than a sequential exploration but - obtains the same optimal parameters + Checks whether sequential and parallel model selection agree on the best parameters. """ - q = PACC(LogisticRegression(random_state=1, max_iter=3000)) - - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=50) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': np.logspace(-3,3,7), 'classifier__class_weight': ['balanced', None]} - app = APP(validation, sample_size=100, random_state=1) + param_grid = {'classifier__C': np.logspace(-3, 3, 7), 'classifier__class_weight': ['balanced', None]} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) def do_gridsearch(n_jobs): - print('starting model selection in sequential exploration') t_init = time.time() modsel = GridSearchQ( - q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, n_jobs=n_jobs, verbose=True - ).fit(*training.Xy) - t_end = time.time()-t_init - best_c = modsel.best_params_['classifier__C'] - print(f'[done] took {t_end:.2f}s best C = {best_c}') - return t_end, best_c + q, param_grid, protocol=app, error='mae', refit=False, timeout=-1, n_jobs=n_jobs, verbose=False + ).fit(*self.training.Xy) + t_end = time.time() - t_init + return t_end, modsel.best_params_ - tend_seq, best_c_seq = do_gridsearch(n_jobs=1) - tend_par, best_c_par = do_gridsearch(n_jobs=-1) - - print(tend_seq, best_c_seq) - print(tend_par, best_c_par) - - self.assertEqual(best_c_seq, best_c_par) - self.assertLess(tend_par, tend_seq) + _, best_seq = do_gridsearch(n_jobs=1) + _, best_par = do_gridsearch(n_jobs=-1) + self.assertEqual(best_seq, best_par) def test_modsel_timeout(self): class SlowLR(LogisticRegression): def fit(self, X, y, sample_weight=None): - import time - time.sleep(10) - super(SlowLR, self).fit(X, y, sample_weight) + time.sleep(2) + return super().fit(X, y, sample_weight) - q = PACC(SlowLR()) + q = PACC(SlowLR(max_iter=1000)) + param_grid = {'classifier__C': np.logspace(-1, 1, 3)} + app = APP(self.validation, sample_size=30, n_prevalences=5, repeats=1, random_state=1) - data = qp.datasets.fetch_reviews('imdb', tfidf=True, min_df=10).reduce(random_state=1) - training, validation = data.training.split_stratified(0.7, random_state=1) - - param_grid = {'classifier__C': np.logspace(-1,1,3)} - app = APP(validation, sample_size=100, random_state=1) - - print('Expecting TimeoutError to be raised') modsel = GridSearchQ( - q, param_grid, protocol=app, timeout=3, n_jobs=-1, verbose=True, raise_errors=True + q, param_grid, protocol=app, timeout=1, n_jobs=-1, verbose=False, raise_errors=True ) with self.assertRaises(TimeoutError): - modsel.fit(*training.Xy) + modsel.fit(*self.training.Xy) - print('Expecting ValueError to be raised') modsel = GridSearchQ( - q, param_grid, protocol=app, timeout=3, n_jobs=-1, verbose=True, raise_errors=False + q, param_grid, protocol=app, timeout=1, n_jobs=-1, verbose=False, raise_errors=False ) with self.assertRaises(ValueError): - # this exception is not raised because of the timeout, but because no combination of hyperparams - # succedded (in this case, a ValueError is raised, regardless of "raise_errors" - modsel.fit(*training.Xy) + modsel.fit(*self.training.Xy) if __name__ == '__main__': diff --git a/quapy/tests/test_plot.py b/quapy/tests/test_plot.py new file mode 100644 index 0000000..3f5a5a5 --- /dev/null +++ b/quapy/tests/test_plot.py @@ -0,0 +1,41 @@ +import unittest + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + HAS_MATPLOTLIB = True +except ImportError: + plt = None + HAS_MATPLOTLIB = False + +import numpy as np + +import quapy as qp + + +@unittest.skipUnless(HAS_MATPLOTLIB and qp.plot is not None, 'matplotlib is not available') +class TestPlot(unittest.TestCase): + + def test_plot_simplex_smoke(self): + rng = np.random.default_rng(0) + true_prev = np.array([0.2, 0.3, 0.5]) + cloud = rng.dirichlet(alpha=30 * true_prev, size=50) + + fig, ax = plt.subplots(figsize=(5, 5)) + fig, ax = qp.plot.plot_simplex( + point_layers=[ + {'points': cloud, 'label': 'cloud', 'style': {'s': 8, 'alpha': 0.2}}, + {'points': true_prev, 'label': 'target', 'style': {'s': 50, 'color': 'black'}}, + ], + region_layers=[ + {'fn': lambda p: p[:, 2] >= 0.4, 'label': 'high class-3', 'color': 'green', 'alpha': 0.2}, + ], + density_function=lambda p: np.exp(-25 * np.sum((p - true_prev) ** 2, axis=1)), + class_names=['A', 'B', 'C'], + ax=ax, + ) + + self.assertIs(fig, ax.figure) + self.assertGreaterEqual(len(ax.collections), 2) + plt.close(fig) diff --git a/quapy/tests/test_protocols.py b/quapy/tests/test_protocols.py index 4850bd4..329762a 100644 --- a/quapy/tests/test_protocols.py +++ b/quapy/tests/test_protocols.py @@ -3,7 +3,7 @@ import numpy as np import quapy.functional from quapy.data import LabelledCollection -from quapy.protocol import APP, NPP, UPP, DomainMixer, AbstractStochasticSeededProtocol +from quapy.protocol import APP, NPP, UPP, DomainMixer, AbstractStochasticSeededProtocol, DirichletProtocol def mock_labelled_collection(prefix=''): @@ -138,6 +138,31 @@ class TestProtocols(unittest.TestCase): self.assertNotEqual(samples1, samples2) + def test_dirichlet_replicate(self): + data = mock_labelled_collection() + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=42) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertEqual(samples1, samples2) + + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=0) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertEqual(samples1, samples2) + + def test_dirichlet_not_replicate(self): + data = mock_labelled_collection() + p = DirichletProtocol(data, alpha=[1, 2, 3, 4], sample_size=5, repeats=10, random_state=None) + + samples1 = samples_to_str(p) + samples2 = samples_to_str(p) + + self.assertNotEqual(samples1, samples2) + def test_covariate_shift_replicate(self): dataA = mock_labelled_collection('domA') dataB = mock_labelled_collection('domB') diff --git a/quapy/tests/test_reader.py b/quapy/tests/test_reader.py new file mode 100644 index 0000000..a7ee21c --- /dev/null +++ b/quapy/tests/test_reader.py @@ -0,0 +1,56 @@ +import os +import tempfile +import unittest + +import numpy as np + +from quapy.data.reader import from_text, from_sparse, from_csv, reindex_labels, binarize + + +class TestReader(unittest.TestCase): + + def _write_tmp(self, content, suffix='.txt'): + fd, path = tempfile.mkstemp(suffix=suffix) + with os.fdopen(fd, 'w') as f: + f.write(content) + self.addCleanup(os.remove, path) + return path + + def test_from_text(self): + path = self._write_tmp('1\tthis is positive\n0\tthis is negative\n') + sentences, labels = from_text(path, verbose=0) + self.assertEqual(sentences, ['this is positive', 'this is negative']) + self.assertEqual(labels, [1, 0]) + + def test_from_text_skips_malformed_lines(self): + # a line without a tab separator should be skipped (and warned about), not raise + path = self._write_tmp('1\tgood line\nthis line has no label\n0\tanother good line\n') + sentences, labels = from_text(path, verbose=0) + self.assertEqual(sentences, ['good line', 'another good line']) + self.assertEqual(labels, [1, 0]) + + def test_from_sparse(self): + # format: