diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html index 8a6f07e..10f0751 100644 --- a/docs/build/html/_modules/index.html +++ b/docs/build/html/_modules/index.html @@ -29,7 +29,7 @@ - + @@ -383,7 +383,6 @@
  • quapy.method._threshold_optim
  • quapy.method.aggregative
  • quapy.method.base
  • -
  • quapy.method.composable
  • quapy.method.confidence
  • quapy.method.meta
  • quapy.method.non_aggregative
  • @@ -391,9 +390,7 @@
  • quapy.plot
  • quapy.protocol
  • quapy.util
  • -
  • qunfold.methods.linear.losses
  • -
  • qunfold.methods.linear.representations
  • -
  • qunfold.sklearn
  • +
  • sklearn.utils._metadata_requests
  • diff --git a/docs/build/html/_modules/quapy/classification/neural.html b/docs/build/html/_modules/quapy/classification/neural.html index 871d391..42d1b4b 100644 --- a/docs/build/html/_modules/quapy/classification/neural.html +++ b/docs/build/html/_modules/quapy/classification/neural.html @@ -29,7 +29,7 @@ - + @@ -370,27 +370,27 @@

    Source code for quapy.classification.neural

    -import logging
    -import os
    -from abc import ABCMeta, abstractmethod
    -from pathlib import Path
    +import logging
    +import os
    +from abc import ABCMeta, abstractmethod
    +from pathlib import Path
     
    -import numpy as np
    -import torch
    -import torch.nn as nn
    -import torch.nn.functional as F
    -from sklearn.metrics import accuracy_score, f1_score
    -from torch.nn.utils.rnn import pad_sequence
    -from tqdm import tqdm
    +import numpy as np
    +import torch
    +import torch.nn as nn
    +import torch.nn.functional as F
    +from sklearn.metrics import accuracy_score, f1_score
    +from torch.nn.utils.rnn import pad_sequence
    +from tqdm import tqdm
     
    -import quapy as qp
    -from quapy.data import LabelledCollection
    -from quapy.util import EarlyStop
    +import quapy as qp
    +from quapy.data import LabelledCollection
    +from quapy.util import EarlyStop
     
     
     
    [docs] -class NeuralClassifierTrainer: +class NeuralClassifierTrainer: """ Trains a neural network for text classification. @@ -408,7 +408,7 @@ according to the evaluation in the held-out validation split (default '../checkpoint/classifier_net.dat') """ - def __init__(self, + def __init__(self, net: 'TextClassifierNet', lr=1e-3, weight_decay=0, @@ -443,7 +443,7 @@
    [docs] - def reset_net_params(self, vocab_size, n_classes): + def reset_net_params(self, vocab_size, n_classes): """Reinitialize the network parameters :param vocab_size: the size of the vocabulary @@ -456,7 +456,7 @@
    [docs] - def get_params(self): + def get_params(self): """Get hyper-parameters for this estimator :return: a dictionary with parameter names mapped to their values @@ -466,7 +466,7 @@
    [docs] - def set_params(self, **params): + def set_params(self, **params): """Set the parameters of this trainer and the learner it is training. In this current version, parameter names for the trainer and learner should be disjoint. @@ -492,14 +492,14 @@ @property - def device(self): + def device(self): """ Gets the device in which the network is allocated :return: device """ return next(self.net.parameters()).device - def _train_epoch(self, data, status, pbar, epoch): + def _train_epoch(self, data, status, pbar, epoch): self.net.train() criterion = torch.nn.CrossEntropyLoss() losses, predictions, true_labels = [], [], [] @@ -519,7 +519,7 @@ status["f1"] = f1_score(true_labels, predictions, average='macro') self.__update_progress_bar(pbar, epoch) - def _test_epoch(self, data, status, pbar, epoch): + def _test_epoch(self, data, status, pbar, epoch): self.net.eval() criterion = torch.nn.CrossEntropyLoss() losses, predictions, true_labels = [], [], [] @@ -537,7 +537,7 @@ status["f1"] = f1_score(true_labels, predictions, average='macro') self.__update_progress_bar(pbar, epoch) - def __update_progress_bar(self, pbar, epoch): + def __update_progress_bar(self, pbar, epoch): pbar.set_description(f'[{self.net.__class__.__name__}] training epoch={epoch} ' f'tr-loss={self.status["tr"]["loss"]:.5f} ' f'tr-acc={100 * self.status["tr"]["acc"]:.2f}% ' @@ -549,7 +549,7 @@
    [docs] - def fit(self, instances, labels, val_split=0.3): + def fit(self, instances, labels, val_split=0.3): """ Fits the model according to the given training data. @@ -599,7 +599,7 @@
    [docs] - def predict(self, instances): + def predict(self, instances): """ Predicts labels for the instances @@ -612,7 +612,7 @@
    [docs] - def predict_proba(self, instances): + def predict_proba(self, instances): """ Predicts posterior probabilities for the instances @@ -631,7 +631,7 @@
    [docs] - def transform(self, instances): + def transform(self, instances): """ Returns the embeddings of the instances @@ -653,7 +653,7 @@
    [docs] -class TorchDataset(torch.utils.data.Dataset): +class TorchDataset(torch.utils.data.Dataset): """ Transforms labelled instances into a Torch's :class:`torch.utils.data.DataLoader` object @@ -661,19 +661,19 @@ :param labels: array-like of shape `(n_samples, n_classes)` with the class labels """ - def __init__(self, instances, labels=None): + def __init__(self, instances, labels=None): self.instances = instances self.labels = labels - def __len__(self): + def __len__(self): return len(self.instances) - def __getitem__(self, index): + def __getitem__(self, index): return {'doc': self.instances[index], 'label': self.labels[index] if self.labels is not None else None}
    [docs] - def asDataloader(self, batch_size, shuffle, pad_length, device): + def asDataloader(self, batch_size, shuffle, pad_length, device): """ Converts the labelled collection into a Torch DataLoader with dynamic padding for the batch @@ -686,7 +686,7 @@ :param device: whether to allocate tensors in cpu or in cuda :return: a :class:`torch.utils.data.DataLoader` object """ - def collate(batch): + def collate(batch): data = [torch.LongTensor(item['doc'][:pad_length]) for item in batch] data = pad_sequence(data, batch_first=True, padding_value=qp.environ['PAD_INDEX']).to(device) targets = [item['label'] for item in batch] @@ -704,7 +704,7 @@
    [docs] -class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta): +class TextClassifierNet(torch.nn.Module, metaclass=ABCMeta): """ Abstract Text classifier (`torch.nn.Module`) """ @@ -712,7 +712,7 @@
    [docs] @abstractmethod - def document_embedding(self, x): + def document_embedding(self, x): """Embeds documents (i.e., performs the forward pass up to the next-to-last layer). @@ -727,7 +727,7 @@
    [docs] - def forward(self, x): + def forward(self, x): """Performs the forward pass. :param x: a batch of instances, typically generated by a torch's `DataLoader` @@ -741,7 +741,7 @@
    [docs] - def dimensions(self): + def dimensions(self): """Gets the number of dimensions of the embedding space :return: integer @@ -751,7 +751,7 @@
    [docs] - def predict_proba(self, x): + def predict_proba(self, x): """ Predicts posterior probabilities for the instances in `x` @@ -766,7 +766,7 @@
    [docs] - def xavier_uniform(self): + def xavier_uniform(self): """ Performs Xavier initialization of the network parameters """ @@ -778,7 +778,7 @@
    [docs] @abstractmethod - def get_params(self): + def get_params(self): """ Get hyper-parameters for this estimator @@ -788,7 +788,7 @@ @property - def vocabulary_size(self): + def vocabulary_size(self): """ Return the size of the vocabulary @@ -800,7 +800,7 @@
    [docs] -class LSTMnet(TextClassifierNet): +class LSTMnet(TextClassifierNet): """ An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on Long Short Term Memory networks. @@ -814,7 +814,7 @@ :param drop_p: drop probability for dropout (default 0.5) """ - def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1, + def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, lstm_class_nlayers=1, drop_p=0.5): super().__init__() @@ -836,7 +836,7 @@ self.doc_embedder = torch.nn.Linear(hidden_size, self.dim) self.output = torch.nn.Linear(self.dim, n_classes) - def __init_hidden(self, set_size): + def __init_hidden(self, set_size): opt = self.hyperparams var_hidden = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size']) var_cell = torch.zeros(opt['lstm_class_nlayers'], set_size, opt['hidden_size']) @@ -846,7 +846,7 @@
    [docs] - def document_embedding(self, x): + def document_embedding(self, x): """Embeds documents (i.e., performs the forward pass up to the next-to-last layer). @@ -865,7 +865,7 @@
    [docs] - def get_params(self): + def get_params(self): """ Get hyper-parameters for this estimator @@ -875,7 +875,7 @@ @property - def vocabulary_size(self): + def vocabulary_size(self): """ Return the size of the vocabulary @@ -887,7 +887,7 @@
    [docs] -class CNNnet(TextClassifierNet): +class CNNnet(TextClassifierNet): """ An implementation of :class:`quapy.classification.neural.TextClassifierNet` based on Convolutional Neural Networks. @@ -904,7 +904,7 @@ :param drop_p: drop probability for dropout (default 0.5) """ - def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, + def __init__(self, vocabulary_size, n_classes, embedding_size=100, hidden_size=256, repr_size=100, kernel_heights=[3, 5, 7], stride=1, padding=0, drop_p=0.5): super(CNNnet, self).__init__() @@ -929,7 +929,7 @@ self.doc_embedder = torch.nn.Linear(len(kernel_heights) * hidden_size, self.dim) self.output = nn.Linear(self.dim, n_classes) - def __conv_block(self, input, conv_layer): + def __conv_block(self, input, conv_layer): conv_out = conv_layer(input) # conv_out.size() = (batch_size, out_channels, dim, 1) activation = F.relu(conv_out.squeeze(3)) # activation.size() = (batch_size, out_channels, dim1) max_out = F.max_pool1d(activation, activation.size()[2]).squeeze(2) # maxpool_out.size() = (batch_size, out_channels) @@ -937,7 +937,7 @@
    [docs] - def document_embedding(self, input): + def document_embedding(self, input): """Embeds documents (i.e., performs the forward pass up to the next-to-last layer). @@ -962,7 +962,7 @@
    [docs] - def get_params(self): + def get_params(self): """ Get hyper-parameters for this estimator @@ -972,7 +972,7 @@ @property - def vocabulary_size(self): + def vocabulary_size(self): """ Return the size of the vocabulary diff --git a/docs/build/html/_modules/quapy/classification/svmperf.html b/docs/build/html/_modules/quapy/classification/svmperf.html index e855546..073c664 100644 --- a/docs/build/html/_modules/quapy/classification/svmperf.html +++ b/docs/build/html/_modules/quapy/classification/svmperf.html @@ -29,7 +29,7 @@ - + @@ -370,22 +370,22 @@

    Source code for quapy.classification.svmperf

    -import logging
    -import random
    -import shutil
    -import subprocess
    -import tempfile
    -from os import remove, makedirs
    -from os.path import join, exists
    -from subprocess import PIPE, STDOUT
    -import numpy as np
    -from sklearn.base import BaseEstimator, ClassifierMixin
    -from sklearn.datasets import dump_svmlight_file
    +import logging
    +import random
    +import shutil
    +import subprocess
    +import tempfile
    +from os import remove, makedirs
    +from os.path import join, exists
    +from subprocess import PIPE, STDOUT
    +import numpy as np
    +from sklearn.base import BaseEstimator, ClassifierMixin
    +from sklearn.datasets import dump_svmlight_file
     
     
     
    [docs] -class SVMperf(BaseEstimator, ClassifierMixin): +class SVMperf(BaseEstimator, ClassifierMixin): """A wrapper for the `SVM-perf package <https://www.cs.cornell.edu/people/tj/svm_light/svm_perf.html>`__ by Thorsten Joachims. When using losses for quantification, the source code has to be patched. See the `installation documentation <https://hlt-isti.github.io/QuaPy/build/html/Installation.html#svm-perf-with-quantification-oriented-losses>`__ @@ -407,7 +407,7 @@ # losses with their respective codes in svm_perf implementation valid_losses = {'01':0, 'f1':1, 'kld':12, 'nkld':13, 'q':22, 'qacc':23, 'qf1':24, 'qgm':25, 'mae':26, 'mrae':27} - def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None): + def __init__(self, svmperf_base, C=0.01, verbose=False, loss='01', host_folder=None): assert exists(svmperf_base), \ (f'path {svmperf_base} does not seem to point to a valid path;' f'did you install svm-perf? ' @@ -420,7 +420,7 @@
    [docs] - def fit(self, X, y): + def fit(self, X, y): """ Trains the SVM for the multivariate performance loss @@ -469,7 +469,7 @@
    [docs] - def predict(self, X): + def predict(self, X): """ Predicts labels for the instances `X` @@ -484,7 +484,7 @@
    [docs] - def decision_function(self, X, y=None): + def decision_function(self, X, y=None): """ Evaluate the decision function for the samples in `X`. @@ -520,7 +520,7 @@ return scores
    - def __del__(self): + def __del__(self): if hasattr(self, 'tmpdir'): shutil.rmtree(self.tmpdir, ignore_errors=True)
    diff --git a/docs/build/html/_modules/quapy/data/base.html b/docs/build/html/_modules/quapy/data/base.html index 87203ec..7a4018c 100644 --- a/docs/build/html/_modules/quapy/data/base.html +++ b/docs/build/html/_modules/quapy/data/base.html @@ -29,7 +29,7 @@ - + @@ -370,23 +370,23 @@

    Source code for quapy.data.base

    -import itertools
    -from functools import cached_property
    -from typing import Iterable
    +import itertools
    +from functools import cached_property
    +from typing import Iterable
     
    -import numpy as np
    -from scipy.sparse import issparse
    -from scipy.sparse import vstack
    -from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
    -from numpy.random import RandomState
    -from quapy.functional import strprev
    -from quapy.util import temp_seed
    -import quapy.functional as F
    +import numpy as np
    +from scipy.sparse import issparse
    +from scipy.sparse import vstack
    +from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
    +from numpy.random import RandomState
    +from quapy.functional import strprev
    +from quapy.util import temp_seed
    +import quapy.functional as F
     
     
     
    [docs] -class LabelledCollection: +class LabelledCollection: """ A LabelledCollection is a set of objects each with a label attached to each of them. This class implements several sampling routines and other utilities. @@ -398,7 +398,7 @@ (i.e., a prevalence of 0) """ - def __init__(self, instances, labels, classes=None): + def __init__(self, instances, labels, classes=None): if issparse(instances): self.instances = instances elif isinstance(instances, list) and len(instances) > 0 and isinstance(instances[0], str): @@ -417,7 +417,7 @@ self._index = None @property - def index(self): + def index(self): if not hasattr(self, '_index') or self._index is None: self._index = {class_: np.arange(len(self))[self.labels == class_] for class_ in self.classes_} return self._index @@ -425,7 +425,7 @@
    [docs] @classmethod - def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs): + def load(cls, path: str, loader_func: callable, classes=None, **loader_kwargs): """ Loads a labelled set of data and convert it into a :class:`LabelledCollection` instance. The function in charge of reading the instances must be specified. This function can be a custom one, or any of the reading functions @@ -442,7 +442,7 @@ return LabelledCollection(*loader_func(path, **loader_kwargs), classes)
    - def __len__(self): + def __len__(self): """ Returns the length of this collection (number of labelled instances) @@ -452,7 +452,7 @@
    [docs] - def prevalence(self): + def prevalence(self): """ Returns the prevalence, or relative frequency, of the classes in the codeframe. @@ -464,7 +464,7 @@
    [docs] - def counts(self): + def counts(self): """ Returns the number of instances for each of the classes in the codeframe. @@ -475,7 +475,7 @@ @property - def n_classes(self): + def n_classes(self): """ The number of classes @@ -484,7 +484,7 @@ return len(self.classes_) @property - def n_instances(self): + def n_instances(self): """ The number of instances @@ -493,7 +493,7 @@ return len(self.labels) @property - def binary(self): + def binary(self): """ Returns True if the number of classes is 2 @@ -503,7 +503,7 @@
    [docs] - def sampling_index(self, size, *prevs, shuffle=True, random_state=None): + def sampling_index(self, size, *prevs, shuffle=True, random_state=None): """ Returns an index to be used to extract a random sample of desired size and desired prevalence values. If the prevalence values are not specified, then returns the index of a uniform sampling. @@ -566,7 +566,7 @@
    [docs] - def uniform_sampling_index(self, size, random_state=None): + def uniform_sampling_index(self, size, random_state=None): """ Returns an index to be used to extract a uniform sample of desired size. The sampling is drawn with replacement. @@ -584,7 +584,7 @@
    [docs] - def sampling(self, size, *prevs, shuffle=True, random_state=None): + def sampling(self, size, *prevs, shuffle=True, random_state=None): """ Return a random sample (an instance of :class:`LabelledCollection`) of desired size and desired prevalence values. For each class, the sampling is drawn with replacement. @@ -604,7 +604,7 @@
    [docs] - def uniform_sampling(self, size, random_state=None): + def uniform_sampling(self, size, random_state=None): """ Returns a uniform sample (an instance of :class:`LabelledCollection`) of desired size. The sampling is drawn with replacement. @@ -619,7 +619,7 @@
    [docs] - def sampling_from_index(self, index): + def sampling_from_index(self, index): """ Returns an instance of :class:`LabelledCollection` whose elements are sampled from this collection using the index. @@ -634,7 +634,7 @@
    [docs] - def split_stratified(self, train_prop=0.6, random_state=None): + def split_stratified(self, train_prop=0.6, random_state=None): """ Returns two instances of :class:`LabelledCollection` split with stratification from this collection, at desired proportion. @@ -656,7 +656,7 @@
    [docs] - def split_random(self, train_prop=0.6, random_state=None): + def split_random(self, train_prop=0.6, random_state=None): """ Returns two instances of :class:`LabelledCollection` split randomly from this collection, at desired proportion. @@ -683,7 +683,7 @@ return training, test
    - def __add__(self, other): + def __add__(self, other): """ Returns a new :class:`LabelledCollection` as the union of this collection with another collection. Both labelled collections must have the same classes. @@ -699,7 +699,7 @@
    [docs] @classmethod - def join(cls, *args: Iterable['LabelledCollection']): + def join(cls, *args: Iterable['LabelledCollection']): """ Returns a new :class:`LabelledCollection` as the union of the collections given in input. @@ -742,7 +742,7 @@ @property - def classes(self): + def classes(self): """ Gets an array-like with the classes used in this collection @@ -751,7 +751,7 @@ return self.classes_ @property - def Xy(self): + def Xy(self): """ Gets the instances and labels. This is useful when working with `sklearn` estimators, e.g.: @@ -762,7 +762,7 @@ return self.instances, self.labels @property - def Xp(self): + def Xp(self): """ Gets the instances and the true prevalence. This is useful when implementing evaluation protocols from a :class:`LabelledCollection` object. @@ -772,7 +772,7 @@ return self.instances, self.prevalence() @property - def X(self): + def X(self): """ An alias to self.instances @@ -781,7 +781,7 @@ return self.instances @property - def y(self): + def y(self): """ An alias to self.labels @@ -790,7 +790,7 @@ return self.labels @property - def p(self): + def p(self): """ An alias to self.prevalence() @@ -801,7 +801,7 @@
    [docs] - def stats(self, show=True): + def stats(self, show=True): """ Returns (and eventually prints) a dictionary with some stats of this collection. E.g.,: @@ -836,7 +836,7 @@
    [docs] - def kFCV(self, nfolds=5, nrepeats=1, random_state=None): + def kFCV(self, nfolds=5, nrepeats=1, random_state=None): """ Generator of stratified folds to be used in k-fold cross validation. @@ -852,7 +852,7 @@ yield train, test
    - def __repr__(self): + def __repr__(self): repr=f'<{self.n_instances} instances (dtype={type(self.instances[0])}), ' repr+=f'n_classes={self.n_classes} {self.classes_}, prevalence={F.strprev(self.prevalence())}>' return repr
    @@ -861,7 +861,7 @@
    [docs] -class Dataset: +class Dataset: """ Abstraction of training and test :class:`LabelledCollection` objects. @@ -871,7 +871,7 @@ :param name: a string representing the name of the dataset """ - def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''): + def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''): assert set(training.classes_) == set(test.classes_), 'incompatible labels in training and test collections' self.training = training self.test = test @@ -881,7 +881,7 @@
    [docs] @classmethod - def SplitStratified(cls, collection: LabelledCollection, train_size=0.6): + def SplitStratified(cls, collection: LabelledCollection, train_size=0.6): """ Generates a :class:`Dataset` from a stratified split of a :class:`LabelledCollection` instance. See :meth:`LabelledCollection.split_stratified` @@ -894,7 +894,7 @@ @property - def classes_(self): + def classes_(self): """ The classes according to which the training collection is labelled @@ -903,7 +903,7 @@ return self.training.classes_ @property - def n_classes(self): + def n_classes(self): """ The number of classes according to which the training collection is labelled @@ -912,7 +912,7 @@ return self.training.n_classes @property - def binary(self): + def binary(self): """ Returns True if the training collection is labelled according to two classes @@ -923,7 +923,7 @@
    [docs] @classmethod - def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs): + def load(cls, train_path, test_path, loader_func: callable, classes=None, **loader_kwargs): """ Loads a training and a test labelled set of data and convert it into a :class:`Dataset` instance. The function in charge of reading the instances must be specified. This function can be a custom one, or any of @@ -945,7 +945,7 @@ @property - def vocabulary_size(self): + def vocabulary_size(self): """ If the dataset is textual, and the vocabulary was indicated, returns the size of the vocabulary @@ -954,7 +954,7 @@ return len(self.vocabulary) @property - def train_test(self): + def train_test(self): """ Alias to `self.training` and `self.test` @@ -965,7 +965,7 @@
    [docs] - def stats(self, show=True): + def stats(self, show=True): """ Returns (and eventually prints) a dictionary with some stats of this dataset. E.g.,: @@ -992,7 +992,7 @@
    [docs] @classmethod - def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0): + def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0): """ Generator of stratified folds to be used in k-fold cross validation. This function is only a wrapper around :meth:`LabelledCollection.kFCV` that returns :class:`Dataset` instances made of training and test folds. @@ -1009,7 +1009,7 @@
    [docs] - def reduce(self, n_train=100, n_test=100, random_state=None): + def reduce(self, n_train=100, n_test=100, random_state=None): """ Reduce the number of instances in place for quick experiments. Preserves the prevalence of each set. @@ -1030,7 +1030,7 @@ return self
    - def __repr__(self): + def __repr__(self): return f'training={self.training}; test={self.test}'
    diff --git a/docs/build/html/_modules/quapy/data/datasets.html b/docs/build/html/_modules/quapy/data/datasets.html index cc6a18b..a35e6ab 100644 --- a/docs/build/html/_modules/quapy/data/datasets.html +++ b/docs/build/html/_modules/quapy/data/datasets.html @@ -29,7 +29,7 @@ - + @@ -370,28 +370,28 @@

    Source code for quapy.data.datasets

    -import logging
    -import os
    -from contextlib import contextmanager
    -import zipfile
    -from os.path import join
    -import pandas as pd
    -from quapy.data.base import Dataset, LabelledCollection
    -from quapy.data.preprocessing import text2tfidf, reduce_columns
    -from quapy.data.preprocessing import standardize as standardizer
    -from quapy.data.reader import *
    -from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
    -from sklearn.preprocessing import StandardScaler
    +import logging
    +import os
    +from contextlib import contextmanager
    +import zipfile
    +from os.path import join
    +import pandas as pd
    +from quapy.data.base import Dataset, LabelledCollection
    +from quapy.data.preprocessing import text2tfidf, reduce_columns
    +from quapy.data.preprocessing import standardize as standardizer
    +from quapy.data.reader import *
    +from quapy.util import download_file_if_not_exists, download_file, get_quapy_home, pickled_resource
    +from sklearn.preprocessing import StandardScaler
     
     
    -def _fetch_ucirepo(*args, **kwargs):
    +def _fetch_ucirepo(*args, **kwargs):
         try:
    -        from ucimlrepo import fetch_ucirepo
    +        from ucimlrepo import fetch_ucirepo
         except ImportError as exc:
             raise ImportError(
                 "UCI dataset fetching requires the optional 'ucimlrepo' package. "
                 "Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset."
    -        ) from exc
    +        ) from exc
         return fetch_ucirepo(*args, **kwargs)
     
     
    @@ -498,7 +498,7 @@
     
     
    [docs] -def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset: +def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset: """ Loads a Reviews dataset as a Dataset instance, as used in `Esuli, A., Moreo, A., and Sebastiani, F. "A recurrent neural network for sentiment quantification." @@ -548,7 +548,7 @@
    [docs] -def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset: +def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset: """ Loads a Twitter dataset as a :class:`quapy.data.base.Dataset` instance, as used in: `Gao, W., Sebastiani, F.: From classification to quantification in tweet sentiment analysis. @@ -626,7 +626,7 @@
    [docs] -def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, standardize=True, verbose=False) -> Dataset: +def fetch_UCIBinaryDataset(dataset_name, data_home=None, test_split=0.3, standardize=True, verbose=False) -> Dataset: """ Loads a UCI dataset as an instance of :class:`quapy.data.base.Dataset`, as used in `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017). @@ -660,7 +660,7 @@
    [docs] -def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize=True, verbose=False) -> LabelledCollection: +def fetch_UCIBinaryLabelledCollection(dataset_name, data_home=None, standardize=True, verbose=False) -> LabelledCollection: """ Loads a UCI collection as an instance of :class:`quapy.data.base.LabelledCollection`, as used in `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017). @@ -853,7 +853,7 @@ file = join(data_home, "uci_datasets", dataset_group + ".pkl") @contextmanager - def download_tmp_file(url_group: str, filename: str): + def download_tmp_file(url_group: str, filename: str): """ Download a data file for a group of datasets temporarely. When used as a context, the file is removed once the context exits. @@ -871,7 +871,7 @@ finally: os.remove(data_path) - def download(id: int | None, group: str) -> dict: + def download(id: int | None, group: str) -> dict: """ Download the data to be pickled for a dataset group. Use the `fetch_ucirepo` api when possible. @@ -937,7 +937,7 @@ return data - def binarize_data(name, data: dict) -> LabelledCollection: + def binarize_data(name, data: dict) -> LabelledCollection: """ Filter and transform data to extract a binary dataset. @@ -982,7 +982,7 @@
    [docs] -def fetch_UCIMulticlassDataset( +def fetch_UCIMulticlassDataset( dataset_name, data_home=None, min_test_split=0.3, @@ -1045,7 +1045,7 @@
    [docs] -def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_class_support=100, standardize=True, verbose=False) -> LabelledCollection: +def fetch_UCIMulticlassLabelledCollection(dataset_name, data_home=None, min_class_support=100, standardize=True, verbose=False) -> LabelledCollection: """ Loads a UCI multiclass collection as an instance of :class:`quapy.data.base.LabelledCollection`. @@ -1148,7 +1148,7 @@ file = join(data_home, 'uci_multiclass', dataset_name+'.pkl') - def dummify_categorical_features(df_features, dataset_id): + def dummify_categorical_features(df_features, dataset_id): categorical_features = { 158: ["S1", "C1", "S2", "C2", "S3", "C3", "S4", "C4", "S5", "C5"], # poker_hand } @@ -1162,7 +1162,7 @@ return X - def download(id, name): + def download(id, name): df = _fetch_ucirepo(id=id) X_df = dummify_categorical_features(df.data.features, id) @@ -1175,7 +1175,7 @@ y = np.searchsorted(classes, y) return LabelledCollection(X, y) - def filter_classes(data: LabelledCollection, min_class_support): + def filter_classes(data: LabelledCollection, min_class_support): if min_class_support is None or min_class_support == 0.: return data if isinstance(min_class_support, float): @@ -1209,18 +1209,18 @@ -def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float): +def _df_replace(df, col, repl={'yes': 1, 'no':0}, astype=float): df[col] = df[col].apply(lambda x:repl[x]).astype(astype, copy=False) -def _array_replace(arr, repl={"yes": 1, "no": 0}): +def _array_replace(arr, repl={"yes": 1, "no": 0}): for k, v in repl.items(): arr[arr == k] = v
    [docs] -def fetch_lequa2022(task, data_home=None): +def fetch_lequa2022(task, data_home=None): """ Loads the official datasets provided for the `LeQua 2022 <https://lequa2022.github.io/index>`_ competition. In brief, there are 4 tasks (T1A, T1B, T2A, T2B) having to do with text quantification @@ -1246,7 +1246,7 @@ that return a series of samples stored in a directory which are labelled by prevalence. """ - from quapy.data._lequa import load_raw_documents, load_vector_documents_2022, SamplesFromDir + from quapy.data._lequa import load_raw_documents, load_vector_documents_2022, SamplesFromDir assert task in LEQUA2022_TASKS, \ f'Unknown task {task}. Valid ones are {LEQUA2022_TASKS}' @@ -1260,7 +1260,7 @@ lequa_dir = join(data_home, 'lequa2022') os.makedirs(lequa_dir, exist_ok=True) - def download_unzip_and_remove(unzipped_path, url): + def download_unzip_and_remove(unzipped_path, url): tmp_path = join(lequa_dir, task + '_tmp.zip') download_file_if_not_exists(url, tmp_path) with zipfile.ZipFile(tmp_path) as file: @@ -1296,7 +1296,7 @@
    [docs] -def fetch_lequa2024(task, data_home=None, merge_T3=False): +def fetch_lequa2024(task, data_home=None, merge_T3=False): """ Loads the official datasets provided for the `LeQua 2024 <https://lequa2024.github.io/index>`_ competition. LeQua 2024 defines four tasks (T1, T2, T3, T4) related to the problem of quantification; @@ -1327,7 +1327,7 @@ that return a series of samples stored in a directory which are labelled by prevalence. """ - from quapy.data._lequa import load_vector_documents_2024, SamplesFromDir, LabelledCollectionsFromDir + from quapy.data._lequa import load_vector_documents_2024, SamplesFromDir, LabelledCollectionsFromDir assert task in LEQUA2024_TASKS, \ f'Unknown task {task}. Valid ones are {LEQUA2024_TASKS}' @@ -1346,7 +1346,7 @@ lequa_dir = join(data_home, 'lequa2024') os.makedirs(lequa_dir, exist_ok=True) - def download_unzip_and_remove(unzipped_path, url): + def download_unzip_and_remove(unzipped_path, url): tmp_path = join(lequa_dir, task + '_tmp.zip') download_file_if_not_exists(url, tmp_path) with zipfile.ZipFile(tmp_path) as file: @@ -1386,7 +1386,7 @@
    [docs] -def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None): +def fetch_IFCB(single_sample_train=True, for_model_selection=False, data_home=None): """ Loads the IFCB dataset for quantification from `Zenodo <https://zenodo.org/records/10036244>`_ (for more information on this dataset, please follow the zenodo link). @@ -1411,7 +1411,7 @@ i.e., a sampling protocol that returns a series of samples labelled by prevalence. """ - from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split + from quapy.data._ifcb import IFCBTrainSamplesFromDir, IFCBTestSamples, get_sample_list, generate_modelselection_split if data_home is None: data_home = get_quapy_home() @@ -1423,7 +1423,7 @@ ifcb_dir = join(data_home, 'ifcb') os.makedirs(ifcb_dir, exist_ok=True) - def download_unzip_and_remove(unzipped_path, url): + def download_unzip_and_remove(unzipped_path, url): tmp_path = join(ifcb_dir, 'ifcb_tmp.zip') download_file_if_not_exists(url, tmp_path) with zipfile.ZipFile(tmp_path) as file: @@ -1468,7 +1468,7 @@ -def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]: +def _fetch_image_embedding_splits(dataset_name, embedding, data_home=None) -> tuple[LabelledCollection,LabelledCollection,LabelledCollection]: """ Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo <https://zenodo.org/records/21131944>`_. @@ -1498,7 +1498,7 @@ trained_network = dataset_network[dataset_name] - def download_embedding_npz(dataset_name, trained_network, embedding): + def download_embedding_npz(dataset_name, trained_network, embedding): target_file = f'{dataset_name}_{trained_network}_{embedding}.npz' URL = f'https://zenodo.org/records/21131944/files/{target_file}' os.makedirs(join(data_home, 'image'), exist_ok=True) @@ -1519,7 +1519,7 @@
    [docs] -def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset: +def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home=None) -> Dataset: """ Loads an image dataset with pre-generated embeddings. Available datasets include: diff --git a/docs/build/html/_modules/quapy/data/reader.html b/docs/build/html/_modules/quapy/data/reader.html index 617442d..88c6f46 100644 --- a/docs/build/html/_modules/quapy/data/reader.html +++ b/docs/build/html/_modules/quapy/data/reader.html @@ -29,7 +29,7 @@ - + @@ -370,16 +370,16 @@

    Source code for quapy.data.reader

    -import logging
    +import logging
     
    -import numpy as np
    -from scipy.sparse import dok_matrix
    -from tqdm import tqdm
    +import numpy as np
    +from scipy.sparse import dok_matrix
    +from tqdm import tqdm
     
     
     
    [docs] -def from_text(path, encoding='utf-8', verbose=1, class2int=True): +def from_text(path, encoding='utf-8', verbose=1, class2int=True): """ Reads a labelled colletion of documents. File fomart <0 or 1>\t<document>\n @@ -413,7 +413,7 @@
    [docs] -def from_sparse(path): +def from_sparse(path): """ Reads a labelled collection of real-valued instances expressed in sparse format File format <-1 or 0 or 1>[\s col(int):val(float)]\n @@ -422,7 +422,7 @@ :return: a `csr_matrix` containing the instances (rows), and a ndarray containing the labels """ - def split_col_val(col_val): + def split_col_val(col_val): col, val = col_val.split(':') col, val = int(col) - 1, float(val) return col, val @@ -450,7 +450,7 @@
    [docs] -def from_csv(path, encoding='utf-8'): +def from_csv(path, encoding='utf-8'): """ Reads a csv file in which columns are separated by ','. File format <label>,<feat1>,<feat2>,...,<featn>\n @@ -473,7 +473,7 @@
    [docs] -def reindex_labels(y): +def reindex_labels(y): """ Re-indexes a list of labels as a list of indexes, and returns the classnames corresponding to the indexes. E.g.: @@ -496,7 +496,7 @@
    [docs] -def binarize(y, pos_class): +def binarize(y, pos_class): """ Binarizes a categorical array-like collection of labels towards the positive class `pos_class`. E.g.,: diff --git a/docs/build/html/_modules/quapy/functional.html b/docs/build/html/_modules/quapy/functional.html index c275ca4..c2fc0bb 100644 --- a/docs/build/html/_modules/quapy/functional.html +++ b/docs/build/html/_modules/quapy/functional.html @@ -29,7 +29,7 @@ - + @@ -370,17 +370,17 @@

    Source code for quapy.functional

    -import warnings
    -from abc import ABC, abstractmethod
    -from collections import defaultdict
    -from functools import lru_cache
    -from typing import Literal, Union, Callable
    -from numpy.typing import ArrayLike
    +import warnings
    +from abc import ABC, abstractmethod
    +from collections import defaultdict
    +from functools import lru_cache
    +from typing import Literal, Union, Callable
    +from numpy.typing import ArrayLike
     
    -import scipy
    -import numpy as np
    +import scipy
    +import numpy as np
     
    -import quapy as qp
    +import quapy as qp
     
     
     # ------------------------------------------------------------------------------------------
    @@ -389,7 +389,7 @@
     
     
    [docs] -def classes_from_labels(labels): +def classes_from_labels(labels): """ Obtains a np.ndarray with the (sorted) classes :param labels: array-like with the instances' labels @@ -403,7 +403,7 @@
    [docs] -def num_classes_from_labels(labels): +def num_classes_from_labels(labels): """ Obtains the number of classes from an array-like of instance's labels :param labels: array-like with the instances' labels @@ -418,7 +418,7 @@
    [docs] -def counts_from_labels(labels: ArrayLike, classes: ArrayLike) -> np.ndarray: +def counts_from_labels(labels: ArrayLike, classes: ArrayLike) -> np.ndarray: """ Computes the raw count values from a vector of labels. @@ -439,7 +439,7 @@
    [docs] -def prevalence_from_labels(labels: ArrayLike, classes: ArrayLike): +def prevalence_from_labels(labels: ArrayLike, classes: ArrayLike): """ Computes the prevalence values from a vector of labels. @@ -457,7 +457,7 @@
    [docs] -def prevalence_from_probabilities(posteriors: ArrayLike, binarize: bool = False): +def prevalence_from_probabilities(posteriors: ArrayLike, binarize: bool = False): """ Returns a vector of prevalence values from a matrix of posterior probabilities. @@ -481,7 +481,7 @@
    [docs] -def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1) -> int: +def num_prevalence_combinations(n_prevpoints:int, n_classes:int, n_repeats:int=1) -> int: """ Computes the number of valid prevalence combinations in the n_classes-dimensional simplex if `n_prevpoints` equally distant prevalence values are generated and `n_repeats` repetitions are requested. @@ -510,7 +510,7 @@
    [docs] -def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1) -> int: +def get_nprevpoints_approximation(combinations_budget:int, n_classes:int, n_repeats:int=1) -> int: """ Searches for the largest number of (equidistant) prevalence points to define for each of the `n_classes` classes so that the number of valid prevalence values generated as combinations of prevalence points (points in a @@ -538,7 +538,7 @@
    [docs] -def as_binary_prevalence(positive_prevalence: Union[float, ArrayLike], clip_if_necessary: bool=False) -> np.ndarray: +def as_binary_prevalence(positive_prevalence: Union[float, ArrayLike], clip_if_necessary: bool=False) -> np.ndarray: """ Helper that, given a float representing the prevalence for the positive class, returns a np.ndarray of two values representing a binary distribution. @@ -560,7 +560,7 @@
    [docs] -def strprev(prevalences: ArrayLike, prec: int=3) -> str: +def strprev(prevalences: ArrayLike, prec: int=3) -> str: """ Returns a string representation for a prevalence vector. E.g., @@ -577,7 +577,7 @@
    [docs] -def check_prevalence_vector(prevalences: ArrayLike, raise_exception: bool=False, tolerance: float=1e-08, aggr=True): +def check_prevalence_vector(prevalences: ArrayLike, raise_exception: bool=False, tolerance: float=1e-08, aggr=True): """ Checks that `prevalences` is a valid prevalence vector, i.e., it contains values in [0,1] and the values sum up to 1. In other words, verifies that the `prevalences` vectors lies in the @@ -619,7 +619,7 @@
    [docs] -def uniform_prevalence(n_classes): +def uniform_prevalence(n_classes): """ Returns a vector representing the uniform distribution for `n_classes` @@ -635,7 +635,7 @@
    [docs] -def normalize_prevalence(prevalences: ArrayLike, method='l1'): +def normalize_prevalence(prevalences: ArrayLike, method='l1'): """ Normalizes a vector or matrix of prevalence values. The normalization consists of applying a L1 normalization in cases in which the prevalence values are not all-zeros, and to convert the prevalence values into `1/n_classes` in @@ -678,7 +678,7 @@
    [docs] -def l1_norm(prevalences: ArrayLike) -> np.ndarray: +def l1_norm(prevalences: ArrayLike) -> np.ndarray: """ Applies L1 normalization to the `unnormalized_arr` so that it becomes a valid prevalence vector. Zero vectors are mapped onto the uniform distribution. Raises an exception if @@ -704,7 +704,7 @@
    [docs] -def clip(prevalences: ArrayLike) -> np.ndarray: +def clip(prevalences: ArrayLike) -> np.ndarray: """ Clips the values in [0,1] and then applies the L1 normalization. @@ -719,7 +719,7 @@
    [docs] -def projection_simplex_sort(unnormalized_arr: ArrayLike) -> np.ndarray: +def projection_simplex_sort(unnormalized_arr: ArrayLike) -> np.ndarray: """Projects a point onto the probability simplex. The code is adapted from Mathieu Blondel's BSD-licensed @@ -747,7 +747,7 @@
    [docs] -def softmax(prevalences: ArrayLike) -> np.ndarray: +def softmax(prevalences: ArrayLike) -> np.ndarray: """ Applies the softmax function to all vectors even if the original vectors were valid distributions. If you want to leave valid vectors untouched, use condsoftmax instead. @@ -762,7 +762,7 @@
    [docs] -def condsoftmax(prevalences: ArrayLike) -> np.ndarray: +def condsoftmax(prevalences: ArrayLike) -> np.ndarray: """ Applies the softmax function only to vectors that do not represent valid distributions. @@ -787,7 +787,7 @@
    [docs] -def HellingerDistance(P: np.ndarray, Q: np.ndarray) -> float: +def HellingerDistance(P: np.ndarray, Q: np.ndarray) -> float: """ Computes the Hellingher Distance (HD) between (discretized) distributions `P` and `Q`. The HD for two discrete distributions of `k` bins is defined as: @@ -805,7 +805,7 @@
    [docs] -def TopsoeDistance(P: np.ndarray, Q: np.ndarray, epsilon: float=1e-20): +def TopsoeDistance(P: np.ndarray, Q: np.ndarray, epsilon: float=1e-20): """ Topsoe distance between two (discretized) distributions `P` and `Q`. The Topsoe distance for two discrete distributions of `k` bins is defined as: @@ -824,7 +824,7 @@
    [docs] -def get_divergence(divergence: Union[str, Callable]): +def get_divergence(divergence: Union[str, Callable]): """ Guarantees that the divergence received as argument is a function. That is, if this argument is already a callable, then it is returned, if it is instead a string, then tries to instantiate the corresponding @@ -853,7 +853,7 @@
    [docs] -def argmin_prevalence(loss: Callable, +def argmin_prevalence(loss: Callable, n_classes: int, method: Literal["optim_minimize", "linear_search", "ternary_search"]='optim_minimize'): """ @@ -880,7 +880,7 @@
    [docs] -def optim_minimize(loss: Callable, n_classes: int, return_loss=False): +def optim_minimize(loss: Callable, n_classes: int, return_loss=False): """ Searches for the optimal prevalence values, i.e., an `n_classes`-dimensional vector of the (`n_classes`-1)-simplex that yields the smallest lost. This optimization is carried out by means of a constrained search using scipy's @@ -892,7 +892,7 @@ :return: (ndarray) the best prevalence vector found or a tuple which also contains the value of the loss if return_loss=True """ - from scipy import optimize + from scipy import optimize # the initial point is set as the uniform distribution uniform_distribution = uniform_prevalence(n_classes=n_classes) @@ -911,7 +911,7 @@