-from copy import deepcopy
+from copy import deepcopy
-from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
-from sklearn.base import BaseEstimator, clone
-from sklearn.model_selection import cross_val_predict, train_test_split
-import numpy as np
+from sklearn.base import BaseEstimator, clone
+from sklearn.model_selection import cross_val_predict, train_test_split
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils.validation import check_X_y
+import numpy as np
# Wrappers of calibration defined by Alexandari et al. in paper <http://proceedings.mlr.press/v119/alexandari20a.html>
@@ -84,7 +88,20 @@
# see https://github.com/kundajelab/abstention
-[docs]class RecalibratedProbabilisticClassifier:
+
def _require_abstention_calibration():
+
try:
+
from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
+
except ImportError as exc:
+
raise ImportError(
+
"Calibration methods in quapy.classification.calibration require the optional "
+
"'abstention' package."
+
) from exc
+
return NoBiasVectorScaling, TempScaling, VectorScaling
+
+
+
+
[docs]
+
class RecalibratedProbabilisticClassifier:
"""
Abstract class for (re)calibration method from `abstention.calibration`, as defined in
`Alexandari, A., Kundaje, A., & Shrikumar, A. (2020, November). Maximum likelihood with bias-corrected calibration
@@ -94,7 +111,10 @@
pass
-
[docs]class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
+
+
+
[docs]
+
class RecalibratedProbabilisticClassifierBase(BaseEstimator, RecalibratedProbabilisticClassifier):
"""
Applies a (re)calibration method from `abstention.calibration`, as defined in
`Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_.
@@ -110,14 +130,16 @@
:param verbose: whether or not to display information in the standard output
"""
-
def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
+
def __init__(self, classifier, calibrator, val_split=5, n_jobs=None, verbose=False):
self.classifier = classifier
self.calibrator = calibrator
self.val_split = val_split
self.n_jobs = n_jobs
self.verbose = verbose
-
[docs] def fit(self, X, y):
+
+
[docs]
+
def fit(self, X, y):
"""
Fits the calibration for the probabilistic classifier.
@@ -135,7 +157,10 @@
raise ValueError('wrong value for val_split: the proportion of validation documents must be in (0,1)')
return self.fit_tr_val(X, y)
-
[docs] def fit_cv(self, X, y):
+
+
+
[docs]
+
def fit_cv(self, X, y):
"""
Fits the calibration in a cross-validation manner, i.e., it generates posterior probabilities for all
training instances via cross-validation, and then retrains the classifier on all training instances.
@@ -153,7 +178,10 @@
self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[y], posterior_supplied=True)
return self
-
[docs] def fit_tr_val(self, X, y):
+
+
+
[docs]
+
def fit_tr_val(self, X, y):
"""
Fits the calibration in a train/val-split manner, i.e.t, it partitions the training instances into a
training and a validation set, and then uses the training samples to learn classifier which is then used
@@ -171,7 +199,10 @@
self.calibration_function = self.calibrator(posteriors, np.eye(nclasses)[yva], posterior_supplied=True)
return self
-
[docs] def predict(self, X):
+
+
+
[docs]
+
def predict(self, X):
"""
Predicts class labels for the data instances in `X`
@@ -180,7 +211,10 @@
"""
return self.classifier.predict(X)
-
[docs] def predict_proba(self, X):
+
+
+
[docs]
+
def predict_proba(self, X):
"""
Generates posterior probabilities for the data instances in `X`
@@ -190,8 +224,9 @@
posteriors = self.classifier.predict_proba(X)
return self.calibration_function(posteriors)
+
@property
-
def classes_(self):
+
def classes_(self):
"""
Returns the classes on which the classifier has been trained on
@@ -200,7 +235,10 @@
return self.classifier.classes_
-
[docs]class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
+
+
+
[docs]
+
class NBVSCalibration(RecalibratedProbabilisticClassifierBase):
"""
Applies the No-Bias Vector Scaling (NBVS) calibration method from `abstention.calibration`, as defined in
`Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -214,7 +252,8 @@
:param verbose: whether or not to display information in the standard output
"""
-
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
NoBiasVectorScaling, _, _ = _require_abstention_calibration()
self.classifier = classifier
self.calibrator = NoBiasVectorScaling(verbose=verbose)
self.val_split = val_split
@@ -222,7 +261,10 @@
self.verbose = verbose
-
[docs]class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
+
+
+
[docs]
+
class BCTSCalibration(RecalibratedProbabilisticClassifierBase):
"""
Applies the Bias-Corrected Temperature Scaling (BCTS) calibration method from `abstention.calibration`, as defined in
`Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -236,7 +278,8 @@
:param verbose: whether or not to display information in the standard output
"""
-
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
_, TempScaling, _ = _require_abstention_calibration()
self.classifier = classifier
self.calibrator = TempScaling(verbose=verbose, bias_positions='all')
self.val_split = val_split
@@ -244,7 +287,10 @@
self.verbose = verbose
-
[docs]class TSCalibration(RecalibratedProbabilisticClassifierBase):
+
+
+
[docs]
+
class TSCalibration(RecalibratedProbabilisticClassifierBase):
"""
Applies the Temperature Scaling (TS) calibration method from `abstention.calibration`, as defined in
`Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -258,7 +304,8 @@
:param verbose: whether or not to display information in the standard output
"""
-
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
_, TempScaling, _ = _require_abstention_calibration()
self.classifier = classifier
self.calibrator = TempScaling(verbose=verbose)
self.val_split = val_split
@@ -266,7 +313,10 @@
self.verbose = verbose
-
[docs]class VSCalibration(RecalibratedProbabilisticClassifierBase):
+
+
+
[docs]
+
class VSCalibration(RecalibratedProbabilisticClassifierBase):
"""
Applies the Vector Scaling (VS) calibration method from `abstention.calibration`, as defined in
`Alexandari et al. paper <http://proceedings.mlr.press/v119/alexandari20a.html>`_:
@@ -280,13 +330,101 @@
:param verbose: whether or not to display information in the standard output
"""
-
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
def __init__(self, classifier, val_split=5, n_jobs=None, verbose=False):
+
_, _, VectorScaling = _require_abstention_calibration()
self.classifier = classifier
self.calibrator = VectorScaling(verbose=verbose)
self.val_split = val_split
self.n_jobs = n_jobs
self.verbose = verbose
+
+
+
+
[docs]
+
class TemperatureScalingFromLogits(BaseEstimator):
+
"""
+
Calibrates a matrix of logits by learning a temperature-scaling mapping
+
with the calibration methods from `abstention.calibration`.
+
+
This estimator is useful when the inputs are already logits produced by a
+
pretrained classifier, and the goal is to transform them directly into
+
calibrated posterior probabilities without retraining the underlying model.
+
+
:param bias_corrected: if True, uses Bias-Corrected Temperature Scaling
+
(BCTS); otherwise, uses standard Temperature Scaling (TS)
+
:param verbose: whether the underlying calibrator should display progress
+
information
+
"""
+
+
def __init__(self, bias_corrected=False, verbose=False):
+
self.bias_corrected = bias_corrected
+
self.verbose = verbose
+
+
+
[docs]
+
def fit(self, X, y):
+
"""
+
Fits the logits calibrator.
+
+
:param X: array-like of shape `(n_samples, n_classes)` containing
+
logits
+
:param y: array-like of shape `(n_samples,)` containing class labels
+
:return: self
+
"""
+
X, y = check_X_y(X, y)
+
+
self.label_encoder_ = LabelEncoder()
+
y_enc = self.label_encoder_.fit_transform(y)
+
self.classes_ = self.label_encoder_.classes_
+
+
n_classes = len(self.classes_)
+
logits_dim = X.shape[1]
+
if n_classes != logits_dim:
+
raise ValueError(
+
f'mismatch between the number of classes ({n_classes}) and the '
+
f'dimensionality of the logits ({logits_dim})'
+
)
+
+
_, TempScaling, _ = _require_abstention_calibration()
+
calibrator = TempScaling(
+
verbose=self.verbose,
+
bias_positions='all' if self.bias_corrected else [],
+
)
+
self.calibrator_ = calibrator
+
self.calibration_function_ = calibrator(X, np.eye(n_classes)[y_enc])
+
return self
+
+
+
+
[docs]
+
def predict_proba(self, X):
+
"""
+
Converts logits into calibrated posterior probabilities.
+
+
:param X: array-like of shape `(n_samples, n_classes)` containing
+
logits
+
:return: array-like of shape `(n_samples, n_classes)` with calibrated
+
posterior probabilities
+
"""
+
return self.calibration_function_(X)
+
+
+
+
[docs]
+
def predict(self, X):
+
"""
+
Predicts class labels after calibration.
+
+
:param X: array-like of shape `(n_samples, n_classes)` containing
+
logits
+
:return: array-like of shape `(n_samples,)` with class label
+
predictions
+
"""
+
posteriors = self.predict_proba(X)
+
return self.label_encoder_.inverse_transform(np.argmax(posteriors, axis=1))
+
+
diff --git a/docs/build/html/_modules/quapy/classification/methods.html b/docs/build/html/_modules/quapy/classification/methods.html
index 883f802..d921a91 100644
--- a/docs/build/html/_modules/quapy/classification/methods.html
+++ b/docs/build/html/_modules/quapy/classification/methods.html
@@ -1,22 +1,20 @@
+
+
-
quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-
-
+
quapy.classification.methods — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+
+
-
-
-
-
-
-
-
+
+
+
+
+
@@ -42,7 +40,13 @@
@@ -70,14 +74,15 @@
Source code for quapy.classification.methods
-from sklearn.base import BaseEstimator
-from sklearn.decomposition import TruncatedSVD
-from sklearn.linear_model import LogisticRegression
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.decomposition import TruncatedSVD
+from sklearn.linear_model import LogisticRegression
[docs]
-
class LowRankLogisticRegression(BaseEstimator):
+
class LowRankLogisticRegression(BaseEstimator):
"""
An example of a classification method (i.e., an object that implements `fit`, `predict`, and `predict_proba`)
that also generates embedded inputs (i.e., that implements `transform`), as those required for
@@ -91,13 +96,13 @@
`Logistic Regression <https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html>`__ classifier
"""
-
def __init__(self, n_components=100, **kwargs):
+
def __init__(self, n_components=100, **kwargs):
self.n_components = n_components
self.classifier = LogisticRegression(**kwargs)
[docs]
-
def get_params(self):
+
def get_params(self):
"""
Get hyper-parameters for this estimator.
@@ -110,7 +115,7 @@
[docs]
-
def set_params(self, **params):
+
def set_params(self, **params):
"""
Set the parameters of this estimator.
@@ -127,7 +132,7 @@
[docs]
-
def fit(self, X, y):
+
def fit(self, X, y):
"""
Fit the model according to the given training data. The fit consists of
fitting `TruncatedSVD` and then `LogisticRegression` on the low-rank representation.
@@ -148,7 +153,7 @@
[docs]
-
def predict(self, X):
+
def predict(self, X):
"""
Predicts labels for the instances `X` embedded into the low-rank space.
@@ -162,7 +167,7 @@
[docs]
-
def predict_proba(self, X):
+
def predict_proba(self, X):
"""
Predicts posterior probabilities for the instances `X` embedded into the low-rank space.
@@ -175,7 +180,7 @@
+
+
+
+
[docs]
+
class MockClassifierFromPosteriors(BaseEstimator):
+
"""
+
Mock classifier that bypasses classifier training when the input instances
+
are already posterior probabilities produced by a pretrained probabilistic
+
classifier.
+
+
:param X: arrays of shape `(n_samples, n_classes)` are interpreted as posterior probabilities
+
"""
+
+
+
[docs]
+
def fit(self, X, y):
+
self.classes_ = np.sort(np.unique(y))
+
return self
+
+
+
+
[docs]
+
def predict(self, X):
+
return np.argmax(X, axis=1)
+
+
+
+
[docs]
+
def predict_proba(self, X):
+
return X
+
+
diff --git a/docs/build/html/_modules/quapy/classification/svmperf.html b/docs/build/html/_modules/quapy/classification/svmperf.html
index 959ad48..9b60c1a 100644
--- a/docs/build/html/_modules/quapy/classification/svmperf.html
+++ b/docs/build/html/_modules/quapy/classification/svmperf.html
@@ -1,22 +1,20 @@
+
+
-
quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-
-
+
quapy.classification.svmperf — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+
+
-
-
-
-
-
-
-
+
+
+
+
+
@@ -42,7 +40,13 @@
@@ -70,21 +74,21 @@
Source code for quapy.classification.svmperf
-import random
-import shutil
-import subprocess
-import tempfile
-from os import remove, makedirs
-from os.path import join, exists
-from subprocess import PIPE, STDOUT
-import numpy as np
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.datasets import dump_svmlight_file
+import random
+import shutil
+import subprocess
+import tempfile
+from os import remove, makedirs
+from os.path import join, exists
+from subprocess import PIPE, STDOUT
+import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.datasets import dump_svmlight_file
[docs]
-
class SVMperf(BaseEstimator, ClassifierMixin):
+
class SVMperf(BaseEstimator, ClassifierMixin):
"""A wrapper for the `SVM-perf package <https://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html>`__ by Thorsten Joachims.
When using losses for quantification, the source code has to be patched. See
the `installation documentation <https://hlt-isti.github.io/QuaPy/build/html/Installation.html#svm-perf-with-quantification-oriented-losses>`__
@@ -106,31 +110,20 @@
# losses with their respective codes in svm_perf implementation
valid_losses = {'01':0, 'f1':1, 'kld':12, 'nkld':13, 'q':22, 'qacc':23, 'qf1':24, 'qgm':25, 'mae':26, 'mrae':27}
-
def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
-
assert exists(svmperf_base), f'path {svmperf_base} does not seem to point to a valid path'
+
def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None):
+
assert exists(svmperf_base), \
+
(f'path {svmperf_base} does not seem to point to a valid path;'
+
f'did you install svm-perf? '
+
f'see instructions in https://hlt-isti.github.io/QuaPy/manuals/explicit-loss-minimization.html')
self.svmperf_base = svmperf_base
self.C = C
self.verbose = verbose
self.loss = loss
self.host_folder = host_folder
-
# def set_params(self, **parameters):
-
# """
-
# Set the hyper-parameters for svm-perf. Currently, only the `C` and `loss` parameters are supported
-
#
-
# :param parameters: a `**kwargs` dictionary `{'C': <float>}`
-
# """
-
# assert sorted(list(parameters.keys())) == ['C', 'loss'], \
-
# 'currently, only the C and loss parameters are supported'
-
# self.C = parameters.get('C', self.C)
-
# self.loss = parameters.get('loss', self.loss)
-
#
-
# def get_params(self, deep=True):
-
# return {'C': self.C, 'loss': self.loss}
-
[docs]
-
def fit(self, X, y):
+
def fit(self, X, y):
"""
Trains the SVM for the multivariate performance loss
@@ -180,7 +173,7 @@
[docs]
-
def predict(self, X):
+
def predict(self, X):
"""
Predicts labels for the instances `X`
@@ -195,7 +188,7 @@
[docs]
-
def decision_function(self, X, y=None):
+
def decision_function(self, X, y=None):
"""
Evaluate the decision function for the samples in `X`.
@@ -231,7 +224,7 @@
return scores
-
def __del__(self):
+
def __del__(self):
if hasattr(self, 'tmpdir'):
shutil.rmtree(self.tmpdir, ignore_errors=True)
diff --git a/docs/build/html/_modules/quapy/data/base.html b/docs/build/html/_modules/quapy/data/base.html
index e3a2e89..27f34de 100644
--- a/docs/build/html/_modules/quapy/data/base.html
+++ b/docs/build/html/_modules/quapy/data/base.html
@@ -1,22 +1,20 @@
+
+
-
quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.1.8 documentation
-
-
+
quapy.data.base — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation
+
+
-
-
-
-
-
-
-
+
+
+
+
+
@@ -42,7 +40,13 @@
@@ -70,22 +74,23 @@
Source code for quapy.data.base
-import itertools
-from functools import cached_property
-from typing import Iterable
+import itertools
+from functools import cached_property
+from typing import Iterable
-import numpy as np
-from scipy.sparse import issparse
-from scipy.sparse import vstack
-from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
-from numpy.random import RandomState
-from quapy.functional import strprev
-from quapy.util import temp_seed
+import numpy as np
+from scipy.sparse import issparse
+from scipy.sparse import vstack
+from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
+from numpy.random import RandomState
+from quapy.functional import strprev
+from quapy.util import temp_seed
+import quapy.functional as F
[docs]
-
class LabelledCollection:
+
class LabelledCollection:
"""
A LabelledCollection is a set of objects each with a label attached to each of them.
This class implements several sampling routines and other utilities.
@@ -97,7 +102,7 @@
(i.e., a prevalence of 0)
"""
-
def __init__(self, instances, labels, classes=None):
+
def __init__(self, instances, labels, classes=None):
if issparse(instances):
self.instances = instances
elif isinstance(instances, list) and len(instances) > 0 and isinstance(instances[0], str):
@@ -106,21 +111,25 @@
else:
self.instances = np.asarray(instances)
self.labels = np.asarray(labels)
-
n_docs = len(self)
if classes is None:
-
self.classes_ = np.unique(self.labels)
-
self.classes_.sort()
+
self.classes_ = F.classes_from_labels(self.labels)
else:
self.classes_ = np.unique(np.asarray(classes))
self.classes_.sort()
if len(set(self.labels).difference(set(classes))) > 0:
raise ValueError(f'labels ({set(self.labels)}) contain values not included in classes_ ({set(classes)})')
-
self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_}
+
self._index = None
+
+
@property
+
def index(self):
+
if not hasattr(self, '_index') or self._index is None:
+
self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_}
+
return self._index
[docs]
@classmethod
-
def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
+
def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs):
"""
Loads a labelled set of data and convert it into a :class:`LabelledCollection` instance. The function in charge
of reading the instances must be specified. This function can be a custom one, or any of the reading functions
@@ -137,7 +146,7 @@
return LabelledCollection(*loader_func(path, **loader_kwargs), classes)
-
def __len__(self):
+
def __len__(self):
"""
Returns the length of this collection (number of labelled instances)
@@ -147,7 +156,7 @@
[docs]
-
def prevalence(self):
+
def prevalence(self):
"""
Returns the prevalence, or relative frequency, of the classes in the codeframe.
@@ -159,7 +168,7 @@
[docs]
-
def counts(self):
+
def counts(self):
"""
Returns the number of instances for each of the classes in the codeframe.
@@ -170,7 +179,7 @@
@property
-
def n_classes(self):
+
def n_classes(self):
"""
The number of classes
@@ -179,7 +188,16 @@
return len(self.classes_)
@property
-
def binary(self):
+
def n_instances(self):
+
"""
+
The number of instances
+
+
:return: integer
+
"""
+
return len(self.labels)
+
+
@property
+
def binary(self):
"""
Returns True if the number of classes is 2
@@ -189,12 +207,11 @@
[docs]
-
def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
+
def sampling_index(self, size, *prevs, shuffle=True, random_state=None):
"""
Returns an index to be used to extract a random sample of desired size and desired prevalence values. If the
prevalence values are not specified, then returns the index of a uniform sampling.
-
For each class, the sampling is drawn with replacement if the requested prevalence is larger than
-
the actual prevalence of the class, or without replacement otherwise.
+
For each class, the sampling is drawn with replacement.
:param size: integer, the requested size
:param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -209,7 +226,7 @@
if len(prevs) == self.n_classes - 1:
prevs = prevs + (1 - sum(prevs),)
assert len(prevs) == self.n_classes, 'unexpected number of prevalences'
-
assert sum(prevs) == 1, f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
+
assert np.isclose(sum(prevs), 1), f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
# Decide how many instances should be taken for each class in order to satisfy the requested prevalence
# accurately, and the number of instances in the sample (exactly). If int(size * prevs[i]) (which is
@@ -238,7 +255,7 @@
for class_, n_requested in n_requests.items():
n_candidates = len(self.index[class_])
index_sample = self.index[class_][
-
np.random.choice(n_candidates, size=n_requested, replace=(n_requested > n_candidates))
+
np.random.choice(n_candidates, size=n_requested, replace=True)
] if n_requested > 0 else []
indexes_sample.append(index_sample)
@@ -253,11 +270,10 @@
+
return ng.choice(len(self), size, replace=True)
[docs]
-
def sampling(self, size, *prevs, shuffle=True, random_state=None):
+
def sampling(self, size, *prevs, shuffle=True, random_state=None):
"""
Return a random sample (an instance of :class:`LabelledCollection`) of desired size and desired prevalence
-
values. For each class, the sampling is drawn without replacement if the requested prevalence is larger than
-
the actual prevalence of the class, or with replacement otherwise.
+
values. For each class, the sampling is drawn with replacement.
:param size: integer, the requested size
:param prevs: the prevalence for each class; the prevalence value for the last class can be lead empty since
@@ -293,11 +308,10 @@