improving docs and new example for image datasets

README.md

@@ -13,7 +13,7 @@ for facilitating the analysis and interpretation of the experimental results.
 
 ### Last updates:
 
-* Version 0.2.0 is released! major changes can be consulted [here](CHANGE_LOG.txt).
+* Version 0.2.1 is released! major changes can be consulted [here](CHANGE_LOG.txt).
 * The developer API documentation is available [here](https://hlt-isti.github.io/QuaPy/index.html)
 * Manuals are available [here](https://hlt-isti.github.io/QuaPy/manuals.html)
 

docs/source/_static/custom.css

@@ -0,0 +1,53 @@
+.navbar-brand.logo .title.logo__title {
+    display: none;
+}
+
+.navbar-brand img {
+    max-height: 2.2rem;
+    width: auto;
+}
+
+.navbar-brand.logo .title.logo__title {
+    font-size: 1.05rem;
+    line-height: 1.15;
+    white-space: nowrap;
+}
+
+@media (max-width: 1200px) {
+    .bd-header .navbar-header-items {
+        min-width: 0;
+    }
+
+    .bd-header .navbar-header-items__center {
+        overflow-x: auto;
+    }
+
+    .bd-header .bd-navbar-elements {
+        flex-wrap: nowrap;
+    }
+}
+
+.hero-copy {
+    font-size: 1.15rem;
+    line-height: 1.7;
+    max-width: 56rem;
+    margin: 0 0 1.5rem 0;
+}
+
+.landing-grid {
+    margin: 1.2rem 0 2rem 0;
+}
+
+.landing-card {
+    border-radius: 1rem;
+    border: 1px solid var(--pst-color-border, #d0d7de);
+    box-shadow: 0 10px 24px rgba(15, 23, 42, 0.08);
+}
+
+.landing-card .sd-card-title {
+    font-size: 1.1rem;
+}
+
+[data-theme="dark"] .landing-card {
+    box-shadow: 0 10px 24px rgba(0, 0, 0, 0.22);
+}

examples/19.visualizing_simplex.py

@@ -0,0 +1,68 @@
+import numpy as np
+import quapy as qp
+from quapy.method.confidence import ConfidenceIntervals
+
+
+"""
+A minimal example showing how to visualise ternary prevalences on the simplex.
+The plot combines a cloud of posterior triplets, a few reference prevalences, a
+confidence ellipse induced by the cloud, and a smooth density centred around the
+true prevalence.
+"""
+
+rng = np.random.default_rng(0)
+true_prev = np.array([0.20, 0.35, 0.45])
+train_prev = np.array([0.50, 0.30, 0.20])
+pred_prev = np.array([0.18, 0.39, 0.43])
+posterior_cloud = rng.dirichlet(alpha=45 * true_prev, size=250)
+
+point_layers = [
+    {
+        'points': posterior_cloud,
+        'label': 'posterior cloud',
+        'style': {'s': 12, 'alpha': 0.25, 'color': 'steelblue', 'edgecolors': 'none'},
+    },
+    {
+        'points': true_prev,
+        'label': 'true prevalence',
+        'style': {'s': 70, 'color': 'black'},
+    },
+    {
+        'points': pred_prev,
+        'label': 'predicted prevalence',
+        'style': {'s': 70, 'color': 'crimson'},
+    },
+    {
+        'points': train_prev,
+        'label': 'training prevalence',
+        'style': {'s': 70, 'color': 'darkorange'},
+    },
+]
+
+confidence_region = ConfidenceIntervals(posterior_cloud, confidence_level=0.95, bonferroni_correction=True)
+
+region_layers = [
+    {
+        'fn': lambda p: float(p in confidence_region),
+        'label': '95% confidence intervals',
+        'color': 'seagreen',
+        'alpha': 0.15,
+    }
+]
+
+density = lambda p: np.exp(-45 * np.sum((p - true_prev) ** 2, axis=1))
+
+qp.plot.plot_simplex(
+    point_layers=point_layers,
+    region_layers=region_layers,
+    density_function=density,
+    density_color='royalblue',
+    class_names=['class A', 'class B', 'class C'],
+    title='Ternary prevalence visualisation',
+    legend_ncol=3,
+    figsize=(7.2, 5.8),
+    class_name_fontsize=9,
+    title_fontsize=10,
+    legend_fontsize=8,
+    savepath='./plots/simplex_visualization.png',
+)

examples/20.cifar10_quantification.py

@@ -0,0 +1,64 @@
+import quapy as qp
+from quapy.data.datasets import fetch_image_embeddings
+from quapy.method.aggregative import EMQ, RLLS
+from quapy.classification.calibration import TemperatureScalingFromLogits
+from quapy.protocol import UPP
+from sklearn.linear_model import LogisticRegression
+
+
+# This example illustrates how to run experiments with image datasets, in this case with CIFAR10
+# The datasets available in quapy do not consist of raw image files, but are instead pre-generated 
+# embeddings (see the manuals for further information).
+
+if __name__ == '__main__':
+
+    # Let us begin with a typical case in which the embeddings come from the penultimate layer of a neural
+    # model (in this case, a resnet18). We get these representations by specifying embedding='features'
+
+    print('fetching cifar10 embeddings')
+    train, test = fetch_image_embeddings(dataset_name='cifar10', embedding='features').train_test
+
+    print('training:', train)
+    print('test:', test)
+
+    Xtr, ytr = train.Xy
+
+    # let us train an Expectation Maximazion Quantifier (EMQ), aka Maximum Likelihood for Label Shift (MLLS)
+    # using a logistic regressor as the underlying classifier, with Bias Corrected Temperature Scaling (BCTS)
+
+    bcts_emq = EMQ(classifier=LogisticRegression(), calib='bcts', val_split=5)
+
+    print(f'fitting quantifier {bcts_emq}')
+    bcts_emq.fit(Xtr, ytr)
+
+    # we generate many samples exhibiting prior probability shift with the artificial prevalence protocol
+    # (we use the multiclass variant UPP instead of the grid-based APP)
+
+    qp.environ["SAMPLE_SIZE"] = 500  # when the sample size is common to all experiments, it is conveniet to set it once and for all
+    artificial_prev_prot = UPP(test, repeats=200)
+    print('generating 200 test bags of 500 instances each')
+
+    bctsemq_report = qp.evaluation.evaluation_report(bcts_emq, protocol=artificial_prev_prot, error_metrics=['mae', 'mrae'])
+    print(bctsemq_report.mean(numeric_only=True))
+
+    # we could instead use the pre-generated logits of the resnet18
+    print('fetching cifar10 logits')
+    train, test = fetch_image_embeddings(dataset_name='cifar10', embedding='logits').train_test
+    Xtr, ytr = train.Xy
+
+    # in this case, the representations are already classification-related outputs;
+    # we can convert them into (hopefully well-) calibrated outputs via TemperatureScaling
+    
+    print('generating posterior probabilities out of logits via temperature scaling')    
+    emq = EMQ(classifier=TemperatureScalingFromLogits(bias_corrected=True))
+    emq.fit(Xtr, ytr)
+
+    print('generating 200 test bags of 500 instances each')
+    artificial_prev_prot = UPP(test, repeats=200)
+    emq_report = qp.evaluation.evaluation_report(emq, protocol=artificial_prev_prot, error_metrics=['mae', 'mrae'])
+    print(emq_report.mean(numeric_only=True))
+
+
+
+
+

quapy/data/datasets.py

@@ -1113,8 +1113,6 @@ def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tu
     val = LabelledCollection(embedding_dict['val'], labels_dict['val'], classes=train.classes)
     test = LabelledCollection(embedding_dict['test'], labels_dict['test'], classes=train.classes)
 
-    print(f'{len(train)} | {len(val)} | {len(test)} | {train.X.shape[1]} | {train.n_classes} | {train.n_classes}')
-
     return train, val, test
 
 

quapy/method/_helper.py

@@ -0,0 +1,107 @@
+"""
+Internal helper utilities shared by quantification methods.
+"""
+
+import numpy as np
+from sklearn.metrics import confusion_matrix
+from sklearn.preprocessing import LabelEncoder
+
+
+def _get_abstention_calibrators():
+    try:
+        from abstention.calibration import NoBiasVectorScaling, TempScaling, VectorScaling
+    except ImportError as exc:
+        raise ImportError(
+            "Posterior calibration for EMQ requires the optional 'abstention' package."
+        ) from exc
+    return {
+        'nbvs': NoBiasVectorScaling(),
+        'bcts': TempScaling(bias_positions='all'),
+        'ts': TempScaling(),
+        'vs': VectorScaling(),
+    }
+
+
+def _get_cvxpy():
+    try:
+        import cvxpy as cp
+    except ImportError as exc:
+        raise ImportError(
+            "RLLS requires the optional 'cvxpy' package."
+        ) from exc
+    return cp
+
+
+def _labels_to_indices(labels, classes):
+    encoder = LabelEncoder().fit(classes)
+    return encoder.transform(labels)
+
+
+def _rlls_check_mode(mode):
+    valid = {'soft', 'hard'}
+    if mode not in valid:
+        raise ValueError(f'unknown mode {mode!r}; valid ones are {valid}')
+    return mode
+
+
+def _rlls_joint_distribution(posteriors, labels, classes, mode='soft'):
+    mode = _rlls_check_mode(mode)
+    posteriors = np.asarray(posteriors, dtype=float)
+    labels = np.asarray(labels)
+    n_samples, n_classes = posteriors.shape
+    assert n_classes == len(classes), 'wrong number of posterior columns'
+
+    if mode == 'hard':
+        pred = np.argmax(posteriors, axis=1)
+        encoded_labels = _labels_to_indices(labels, classes)
+        joint = confusion_matrix(encoded_labels, pred, labels=np.arange(n_classes)).T.astype(float)
+        return joint / n_samples
+
+    joint = np.zeros((n_classes, n_classes), dtype=float)
+    for class_index, class_ in enumerate(classes):
+        idx = labels == class_
+        if idx.any():
+            joint[:, class_index] = posteriors[idx].sum(axis=0)
+    return joint / n_samples
+
+
+def _rlls_predicted_marginal(posteriors, mode='soft'):
+    mode = _rlls_check_mode(mode)
+    posteriors = np.asarray(posteriors, dtype=float)
+    if mode == 'soft':
+        return posteriors.mean(axis=0)
+
+    pred = np.argmax(posteriors, axis=1)
+    counts = np.bincount(pred, minlength=posteriors.shape[1]).astype(float)
+    return counts / counts.sum()
+
+
+def _rlls_compute_3deltaC(n_classes, n_train, delta):
+    return 3 * (
+        2 * np.log(2 * n_classes / delta) / (3 * n_train)
+        + np.sqrt(2 * np.log(2 * n_classes / delta) / n_train)
+    )
+
+
+def _rlls_compute_weights(C_zy, qz, pz, rho, clip=False):
+    cp = _get_cvxpy()
+
+    n_classes = C_zy.shape[0]
+    theta = cp.Variable(n_classes)
+    b = qz - pz
+    objective = cp.Minimize(cp.pnorm(C_zy @ theta - b) + rho * cp.pnorm(theta))
+    constraints = [-1 <= theta]
+    problem = cp.Problem(objective, constraints)
+
+    try:
+        problem.solve(verbose=False, solver=cp.SCS)
+    except cp.error.SolverError:
+        problem.solve(verbose=False, solver=cp.SCS, use_indirect=True)
+
+    if theta.value is None:
+        raise RuntimeError('RLLS optimization failed to produce a solution')
+
+    w = 1 + np.asarray(theta.value, dtype=float)
+    if clip and np.any(w < 0):
+        w = np.clip(w, 0, None)
+    return w

quapy/tests/test_functional.py

@@ -0,0 +1,27 @@
+import unittest
+
+import numpy as np
+
+import quapy.functional as F
+
+
+class TestFunctional(unittest.TestCase):
+
+    def test_ternary_search_binary(self):
+        def loss(prev):
+            return (prev[1] - 0.37) ** 2
+
+        result = F.argmin_prevalence(loss, n_classes=2, method='ternary_search')
+        self.assertTrue(np.allclose(result.sum(), 1.0))
+        self.assertAlmostEqual(result[1], 0.37, places=3)
+
+    def test_ternary_search_multiclass_not_supported(self):
+        def loss(prev):
+            return np.sum((prev - np.array([0.2, 0.3, 0.5])) ** 2)
+
+        with self.assertRaises(AssertionError):
+            F.argmin_prevalence(loss, n_classes=3, method='ternary_search')
+
+
+if __name__ == '__main__':
+    unittest.main()

quapy/tests/test_plot.py

@@ -0,0 +1,41 @@
+import unittest
+
+try:
+    import matplotlib
+    matplotlib.use('Agg')
+    import matplotlib.pyplot as plt
+    HAS_MATPLOTLIB = True
+except ImportError:
+    plt = None
+    HAS_MATPLOTLIB = False
+
+import numpy as np
+
+import quapy as qp
+
+
+@unittest.skipUnless(HAS_MATPLOTLIB and qp.plot is not None, 'matplotlib is not available')
+class TestPlot(unittest.TestCase):
+
+    def test_plot_simplex_smoke(self):
+        rng = np.random.default_rng(0)
+        true_prev = np.array([0.2, 0.3, 0.5])
+        cloud = rng.dirichlet(alpha=30 * true_prev, size=50)
+
+        fig, ax = plt.subplots(figsize=(5, 5))
+        fig, ax = qp.plot.plot_simplex(
+            point_layers=[
+                {'points': cloud, 'label': 'cloud', 'style': {'s': 8, 'alpha': 0.2}},
+                {'points': true_prev, 'label': 'target', 'style': {'s': 50, 'color': 'black'}},
+            ],
+            region_layers=[
+                {'fn': lambda p: p[:, 2] >= 0.4, 'label': 'high class-3', 'color': 'green', 'alpha': 0.2},
+            ],
+            density_function=lambda p: np.exp(-25 * np.sum((p - true_prev) ** 2, axis=1)),
+            class_names=['A', 'B', 'C'],
+            ax=ax,
+        )
+
+        self.assertIs(fig, ax.figure)
+        self.assertGreaterEqual(len(ax.collections), 2)
+        plt.close(fig)