1
0
Fork 0

Merge pull request #1 from HLT-ISTI/tests_and_classnames

Tests and class names
This commit is contained in:
Andrea Esuli 2021-05-10 10:28:16 +02:00 committed by GitHub
commit c280c03fdb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 346 additions and 124 deletions

View File

@ -17,14 +17,13 @@ Current issues:
In binary quantification (hp, kindle, imdb) we used F1 in the minority class (which in kindle and hp happens to be the
negative class). This is not covered in this new implementation, in which the binary case is not treated as such, but as
an instance of single-label with 2 labels. Check
Add classnames to LabelledCollection? This should improve visualization of reports
Add automatic reindex of class labels in LabelledCollection (currently, class indexes should be ordered and with no gaps)
OVR I believe is currently tied to aggregative methods. We should provide a general interface also for general quantifiers
Currently, being "binary" only adds one checker; we should figure out how to impose the check to be automatically performed
Add random seed management to support replicability (see temp_seed in util.py).
Improvements:
==========================================
Clarify whether QuaNet is an aggregative method or not.
Explore the hyperparameter "number of bins" in HDy
Rename EMQ to SLD ?
Parallelize the kFCV in ACC and PACC?

View File

@ -11,8 +11,8 @@ from torch.nn.utils.rnn import pad_sequence
from tqdm import tqdm
import quapy as qp
from data import LabelledCollection
from util import EarlyStop
from quapy.data import LabelledCollection
from quapy.util import EarlyStop
class NeuralClassifierTrainer:

View File

@ -2,40 +2,52 @@ import numpy as np
from scipy.sparse import issparse
from scipy.sparse import vstack
from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold
from quapy.functional import artificial_prevalence_sampling, strprev
class LabelledCollection:
'''
A LabelledCollection is a set of objects each with a label associated to it.
'''
def __init__(self, instances, labels, n_classes=None):
def __init__(self, instances, labels, classes_=None):
"""
:param instances: list of objects
:param labels: list of labels, same length of instances
:param classes_: optional, list of classes from which labels are taken. When used, must contain the set of values used in labels.
"""
if issparse(instances):
self.instances = instances
elif isinstance(instances, list) and len(instances)>0 and isinstance(instances[0], str):
elif isinstance(instances, list) and len(instances) > 0 and isinstance(instances[0], str):
# lists of strings occupy too much as ndarrays (although python-objects add a heavy overload)
self.instances = np.asarray(instances, dtype=object)
else:
self.instances = np.asarray(instances)
self.labels = np.asarray(labels, dtype=int)
self.labels = np.asarray(labels)
n_docs = len(self)
if n_classes is None:
if classes_ is None:
self.classes_ = np.unique(self.labels)
self.classes_.sort()
else:
self.classes_ = np.arange(n_classes)
self.index = {class_i: np.arange(n_docs)[self.labels == class_i] for class_i in self.classes_}
self.classes_ = np.unique(np.asarray(classes_))
self.classes_.sort()
if len(set(self.labels).difference(set(classes_))) > 0:
raise ValueError('labels contains values not included in classes_')
self.index = {class_: np.arange(n_docs)[self.labels == class_] for class_ in self.classes_}
@classmethod
def load(cls, path:str, loader_func:callable):
def load(cls, path: str, loader_func: callable):
return LabelledCollection(*loader_func(path))
def __len__(self):
return self.instances.shape[0]
def prevalence(self):
return self.counts()/len(self)
return self.counts() / len(self)
def counts(self):
return np.asarray([len(self.index[ci]) for ci in self.classes_])
return np.asarray([len(self.index[class_]) for class_ in self.classes_])
@property
def n_classes(self):
@ -48,21 +60,21 @@ class LabelledCollection:
def sampling_index(self, size, *prevs, shuffle=True):
if len(prevs) == 0: # no prevalence was indicated; returns an index for uniform sampling
return np.random.choice(len(self), size, replace=False)
if len(prevs) == self.n_classes-1:
prevs = prevs + (1-sum(prevs),)
if len(prevs) == self.n_classes - 1:
prevs = prevs + (1 - sum(prevs),)
assert len(prevs) == self.n_classes, 'unexpected number of prevalences'
assert sum(prevs) == 1, f'prevalences ({prevs}) wrong range (sum={sum(prevs)})'
taken = 0
indexes_sample = []
for i, class_i in enumerate(self.classes_):
if i == self.n_classes-1:
for i, class_ in enumerate(self.classes_):
if i == self.n_classes - 1:
n_requested = size - taken
else:
n_requested = int(size * prevs[i])
n_candidates = len(self.index[class_i])
index_sample = self.index[class_i][
n_candidates = len(self.index[class_])
index_sample = self.index[class_][
np.random.choice(n_candidates, size=n_requested, replace=(n_requested > n_candidates))
] if n_requested > 0 else []
@ -90,21 +102,22 @@ class LabelledCollection:
def sampling_from_index(self, index):
documents = self.instances[index]
labels = self.labels[index]
return LabelledCollection(documents, labels, n_classes=self.n_classes)
return LabelledCollection(documents, labels, classes_=self.classes_)
def split_stratified(self, train_prop=0.6, random_state=None):
# with temp_seed(42):
tr_docs, te_docs, tr_labels, te_labels = \
train_test_split(self.instances, self.labels, train_size=train_prop, stratify=self.labels, random_state=random_state)
train_test_split(self.instances, self.labels, train_size=train_prop, stratify=self.labels,
random_state=random_state)
return LabelledCollection(tr_docs, tr_labels), LabelledCollection(te_docs, te_labels)
def artificial_sampling_generator(self, sample_size, n_prevalences=101, repeats=1):
dimensions=self.n_classes
dimensions = self.n_classes
for prevs in artificial_prevalence_sampling(dimensions, n_prevalences, repeats):
yield self.sampling(sample_size, *prevs)
def artificial_sampling_index_generator(self, sample_size, n_prevalences=101, repeats=1):
dimensions=self.n_classes
dimensions = self.n_classes
for prevs in artificial_prevalence_sampling(dimensions, n_prevalences, repeats):
yield self.sampling_index(sample_size, *prevs)
@ -142,10 +155,10 @@ class LabelledCollection:
else:
nfeats = '?'
stats_ = {'instances': ninstances,
'type': instance_type,
'features': nfeats,
'classes': self.n_classes,
'prevs': strprev(self.prevalence())}
'type': instance_type,
'features': nfeats,
'classes': self.classes_,
'prevs': strprev(self.prevalence())}
if show:
print(f'#instances={stats_["instances"]}, type={stats_["type"]}, #features={stats_["features"]}, '
f'#classes={stats_["classes"]}, prevs={stats_["prevs"]}')
@ -155,13 +168,14 @@ class LabelledCollection:
kf = RepeatedStratifiedKFold(n_splits=nfolds, n_repeats=nrepeats, random_state=random_state)
for train_index, test_index in kf.split(*self.Xy):
train = self.sampling_from_index(train_index)
test = self.sampling_from_index(test_index)
test = self.sampling_from_index(test_index)
yield train, test
class Dataset:
def __init__(self, training: LabelledCollection, test: LabelledCollection, vocabulary: dict = None, name=''):
assert training.n_classes == test.n_classes, 'incompatible labels in training and test collections'
assert set(training.classes_) == set(test.classes_), 'incompatible labels in training and test collections'
self.training = training
self.test = test
self.vocabulary = vocabulary
@ -171,6 +185,10 @@ class Dataset:
def SplitStratified(cls, collection: LabelledCollection, train_size=0.6):
return Dataset(*collection.split_stratified(train_prop=train_size))
@property
def classes_(self):
return self.training.classes_
@property
def n_classes(self):
return self.training.n_classes
@ -195,19 +213,15 @@ class Dataset:
print(f'Dataset={self.name} #tr-instances={tr_stats["instances"]}, #te-instances={te_stats["instances"]}, '
f'type={tr_stats["type"]}, #features={tr_stats["features"]}, #classes={tr_stats["classes"]}, '
f'tr-prevs={tr_stats["prevs"]}, te-prevs={te_stats["prevs"]}')
return {'train': tr_stats ,'test':te_stats}
return {'train': tr_stats, 'test': te_stats}
@classmethod
def kFCV(cls, data: LabelledCollection, nfolds=5, nrepeats=1, random_state=0):
for i, (train, test) in enumerate(data.kFCV(nfolds=nfolds, nrepeats=nrepeats, random_state=random_state)):
yield Dataset(train, test, name=f'fold {(i%nfolds)+1}/{nfolds} (round={(i//nfolds)+1})')
yield Dataset(train, test, name=f'fold {(i % nfolds) + 1}/{nfolds} (round={(i // nfolds) + 1})')
def isbinary(data):
if isinstance(data, Dataset) or isinstance(data, LabelledCollection):
return data.binary
return False

View File

@ -47,7 +47,7 @@ UCI_DATASETS = ['acute.a', 'acute.b',
'yeast']
def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False):
def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle=False) -> Dataset:
"""
Load a Reviews dataset as a Dataset instance, as used in:
Esuli, A., Moreo, A., and Sebastiani, F. "A recurrent neural network for sentiment quantification."
@ -91,7 +91,7 @@ def fetch_reviews(dataset_name, tfidf=False, min_df=None, data_home=None, pickle
return data
def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False):
def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_home=None, pickle=False) -> Dataset:
"""
Load a Twitter dataset as a Dataset instance, as used in:
Gao, W., Sebastiani, F.: From classification to quantification in tweet sentiment analysis.
@ -162,12 +162,12 @@ def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_hom
return data
def fetch_UCIDataset(dataset_name, data_home=None, test_split=0.3, verbose=False):
def fetch_UCIDataset(dataset_name, data_home=None, test_split=0.3, verbose=False) -> Dataset:
data = fetch_UCILabelledCollection(dataset_name, data_home, verbose)
return Dataset(*data.split_stratified(1 - test_split, random_state=0))
def fetch_UCILabelledCollection(dataset_name, data_home=None, verbose=False):
def fetch_UCILabelledCollection(dataset_name, data_home=None, verbose=False) -> Dataset:
assert dataset_name in UCI_DATASETS, \
f'Name {dataset_name} does not match any known dataset from the UCI Machine Learning datasets repository. ' \

View File

@ -29,13 +29,13 @@ def text2tfidf(dataset:Dataset, min_df=3, sublinear_tf=True, inplace=False, **kw
test_documents = vectorizer.transform(dataset.test.instances)
if inplace:
dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.n_classes)
dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.n_classes)
dataset.training = LabelledCollection(training_documents, dataset.training.labels, dataset.classes_)
dataset.test = LabelledCollection(test_documents, dataset.test.labels, dataset.classes_)
dataset.vocabulary = vectorizer.vocabulary_
return dataset
else:
training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.n_classes)
test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.n_classes)
training = LabelledCollection(training_documents, dataset.training.labels.copy(), dataset.classes_)
test = LabelledCollection(test_documents, dataset.test.labels.copy(), dataset.classes_)
return Dataset(training, test, vectorizer.vocabulary_)
@ -66,8 +66,8 @@ def reduce_columns(dataset: Dataset, min_df=5, inplace=False):
dataset.test.instances = Xte
return dataset
else:
training = LabelledCollection(Xtr, dataset.training.labels.copy(), dataset.n_classes)
test = LabelledCollection(Xte, dataset.test.labels.copy(), dataset.n_classes)
training = LabelledCollection(Xtr, dataset.training.labels.copy(), dataset.classes_)
test = LabelledCollection(Xte, dataset.test.labels.copy(), dataset.classes_)
return Dataset(training, test)
@ -100,13 +100,13 @@ def index(dataset: Dataset, min_df=5, inplace=False, **kwargs):
test_index = indexer.transform(dataset.test.instances)
if inplace:
dataset.training = LabelledCollection(training_index, dataset.training.labels, dataset.n_classes)
dataset.test = LabelledCollection(test_index, dataset.test.labels, dataset.n_classes)
dataset.training = LabelledCollection(training_index, dataset.training.labels, dataset.classes_)
dataset.test = LabelledCollection(test_index, dataset.test.labels, dataset.classes_)
dataset.vocabulary = indexer.vocabulary_
return dataset
else:
training = LabelledCollection(training_index, dataset.training.labels.copy(), dataset.n_classes)
test = LabelledCollection(test_index, dataset.test.labels.copy(), dataset.n_classes)
training = LabelledCollection(training_index, dataset.training.labels.copy(), dataset.classes_)
test = LabelledCollection(test_index, dataset.test.labels.copy(), dataset.classes_)
return Dataset(training, test, indexer.vocabulary_)

View File

@ -3,7 +3,7 @@ from scipy.sparse import dok_matrix
from tqdm import tqdm
def from_text(path):
def from_text(path, encoding='utf-8'):
"""
Reas a labelled colletion of documents.
File fomart <0 or 1>\t<document>\n
@ -11,7 +11,7 @@ def from_text(path):
:return: a list of sentences, and a list of labels
"""
all_sentences, all_labels = [], []
for line in tqdm(open(path, 'rt').readlines(), f'loading {path}'):
for line in tqdm(open(path, 'rt', encoding=encoding).readlines(), f'loading {path}'):
line = line.strip()
if line:
label, sentence = line.split('\t')
@ -25,8 +25,8 @@ def from_text(path):
def from_sparse(path):
"""
Reas a labelled colletion of real-valued instances expressed in sparse format
File fomart <-1 or 0 or 1>[\s col(int):val(float)]\n
Reads a labelled collection of real-valued instances expressed in sparse format
File format <-1 or 0 or 1>[\s col(int):val(float)]\n
:param path: path to the labelled collection
:return: a csr_matrix containing the instances (rows), and a ndarray containing the labels
"""
@ -56,16 +56,16 @@ def from_sparse(path):
return X, y
def from_csv(path):
def from_csv(path, encoding='utf-8'):
"""
Reas a csv file in which columns are separated by ','.
File fomart <label>,<feat1>,<feat2>,...,<featn>\n
Reads a csv file in which columns are separated by ','.
File format <label>,<feat1>,<feat2>,...,<featn>\n
:param path: path to the csv file
:return: a ndarray for the labels and a ndarray (float) for the covariates
"""
X, y = [], []
for instance in tqdm(open(path, 'rt').readlines(), desc=f'reading {path}'):
for instance in tqdm(open(path, 'rt', encoding=encoding).readlines(), desc=f'reading {path}'):
yi, *xi = instance.strip().split(',')
X.append(list(map(float,xi)))
y.append(yi)

View File

@ -36,12 +36,12 @@ def prevalence_linspace(n_prevalences=21, repeat=1, smooth_limits_epsilon=0.01):
return p
def prevalence_from_labels(labels, n_classes):
def prevalence_from_labels(labels, classes_):
if labels.ndim != 1:
raise ValueError(f'param labels does not seem to be a ndarray of label predictions')
unique, counts = np.unique(labels, return_counts=True)
by_class = defaultdict(lambda:0, dict(zip(unique, counts)))
prevalences = np.asarray([by_class[ci] for ci in range(n_classes)], dtype=np.float)
prevalences = np.asarray([by_class[class_] for class_ in classes_], dtype=np.float)
prevalences /= prevalences.sum()
return prevalences
@ -51,7 +51,7 @@ def prevalence_from_probabilities(posteriors, binarize: bool = False):
raise ValueError(f'param posteriors does not seem to be a ndarray of posteior probabilities')
if binarize:
predictions = np.argmax(posteriors, axis=-1)
return prevalence_from_labels(predictions, n_classes=posteriors.shape[1])
return prevalence_from_labels(predictions, np.arange(posteriors.shape[1]))
else:
prevalences = posteriors.mean(axis=0)
prevalences /= prevalences.sum()

View File

@ -3,21 +3,31 @@ from . import base
from . import meta
from . import non_aggregative
EXPLICIT_LOSS_MINIMIZATION_METHODS = {
aggregative.ELM,
aggregative.SVMQ,
aggregative.SVMAE,
aggregative.SVMKLD,
aggregative.SVMRAE,
aggregative.SVMNKLD
}
AGGREGATIVE_METHODS = {
aggregative.CC,
aggregative.ACC,
aggregative.PCC,
aggregative.PACC,
aggregative.ELM,
aggregative.EMQ,
aggregative.HDy
}
} | EXPLICIT_LOSS_MINIMIZATION_METHODS
NON_AGGREGATIVE_METHODS = {
non_aggregative.MaximumLikelihoodPrevalenceEstimation
}
META_METHODS = {
meta.Ensemble,
meta.QuaNet
}

View File

@ -1,6 +1,7 @@
from abc import abstractmethod
from copy import deepcopy
from typing import Union
import numpy as np
from joblib import Parallel, delayed
from sklearn.base import BaseEstimator
@ -8,6 +9,7 @@ from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm
import quapy as qp
import quapy.functional as F
from quapy.classification.svmperf import SVMperf
@ -43,7 +45,7 @@ class AggregativeQuantifier(BaseQuantifier):
return self.aggregate(classif_predictions)
@abstractmethod
def aggregate(self, classif_predictions:np.ndarray): ...
def aggregate(self, classif_predictions: np.ndarray): ...
def get_params(self, deep=True):
return self.learner.get_params()
@ -53,10 +55,10 @@ class AggregativeQuantifier(BaseQuantifier):
@property
def n_classes(self):
return len(self.classes)
return len(self.classes_)
@property
def classes(self):
def classes_(self):
return self.learner.classes_
@property
@ -84,7 +86,7 @@ class AggregativeProbabilisticQuantifier(AggregativeQuantifier):
def set_params(self, **parameters):
if isinstance(self.learner, CalibratedClassifierCV):
parameters = {'base_estimator__'+k:v for k,v in parameters.items()}
parameters = {'base_estimator__' + k: v for k, v in parameters.items()}
self.learner.set_params(**parameters)
@property
@ -98,7 +100,7 @@ def training_helper(learner,
data: LabelledCollection,
fit_learner: bool = True,
ensure_probabilistic=False,
val_split:Union[LabelledCollection, float]=None):
val_split: Union[LabelledCollection, float] = None):
"""
Training procedure common to all Aggregative Quantifiers.
:param learner: the learner to be fit
@ -122,13 +124,14 @@ def training_helper(learner,
if isinstance(val_split, float):
if not (0 < val_split < 1):
raise ValueError(f'train/val split {val_split} out of range, must be in (0,1)')
train, unused = data.split_stratified(train_prop=1-val_split)
elif val_split.__class__.__name__ == LabelledCollection.__name__: #isinstance(val_split, LabelledCollection):
train, unused = data.split_stratified(train_prop=1 - val_split)
elif val_split.__class__.__name__ == LabelledCollection.__name__: # isinstance(val_split, LabelledCollection):
train = data
unused = val_split
else:
raise ValueError(f'param "val_split" ({type(val_split)}) not understood; use either a float indicating the split '
'proportion, or a LabelledCollection indicating the validation split')
raise ValueError(
f'param "val_split" ({type(val_split)}) not understood; use either a float indicating the split '
'proportion, or a LabelledCollection indicating the validation split')
else:
train, unused = data, None
@ -153,7 +156,7 @@ class CC(AggregativeQuantifier):
attributed each of the classes in order to compute class prevalence estimates.
"""
def __init__(self, learner:BaseEstimator):
def __init__(self, learner: BaseEstimator):
self.learner = learner
def fit(self, data: LabelledCollection, fit_learner=True):
@ -167,16 +170,16 @@ class CC(AggregativeQuantifier):
return self
def aggregate(self, classif_predictions):
return F.prevalence_from_labels(classif_predictions, self.n_classes)
return F.prevalence_from_labels(classif_predictions, self.classes_)
class ACC(AggregativeQuantifier):
def __init__(self, learner:BaseEstimator, val_split=0.4):
def __init__(self, learner: BaseEstimator, val_split=0.4):
self.learner = learner
self.val_split = val_split
def fit(self, data: LabelledCollection, fit_learner=True, val_split: Union[float, int, LabelledCollection]=None):
def fit(self, data: LabelledCollection, fit_learner=True, val_split: Union[float, int, LabelledCollection] = None):
"""
Trains a ACC quantifier
:param data: the training set
@ -262,7 +265,7 @@ class PACC(AggregativeProbabilisticQuantifier):
self.learner = learner
self.val_split = val_split
def fit(self, data: LabelledCollection, fit_learner=True, val_split:Union[float, int, LabelledCollection]=None):
def fit(self, data: LabelledCollection, fit_learner=True, val_split: Union[float, int, LabelledCollection] = None):
"""
Trains a PACC quantifier
:param data: the training set
@ -294,7 +297,8 @@ class PACC(AggregativeProbabilisticQuantifier):
y_ = np.vstack(y_)
# fit the learner on all data
self.learner, _ = training_helper(self.learner, data, fit_learner, ensure_probabilistic=True, val_split=None)
self.learner, _ = training_helper(self.learner, data, fit_learner, ensure_probabilistic=True,
val_split=None)
else:
self.learner, val_data = training_helper(
@ -307,8 +311,8 @@ class PACC(AggregativeProbabilisticQuantifier):
# estimate the matrix with entry (i,j) being the estimate of P(yi|yj), that is, the probability that a
# document that belongs to yj ends up being classified as belonging to yi
confusion = np.empty(shape=(data.n_classes, data.n_classes))
for yi in range(data.n_classes):
confusion[yi] = y_[y==yi].mean(axis=0)
for i,class_ in enumerate(data.classes_):
confusion[i] = y_[y == class_].mean(axis=0)
self.Pte_cond_estim_ = confusion.T
@ -338,7 +342,7 @@ class EMQ(AggregativeProbabilisticQuantifier):
def fit(self, data: LabelledCollection, fit_learner=True):
self.learner, _ = training_helper(self.learner, data, fit_learner, ensure_probabilistic=True)
self.train_prevalence = F.prevalence_from_labels(data.labels, self.n_classes)
self.train_prevalence = F.prevalence_from_labels(data.labels, self.classes_)
return self
def aggregate(self, classif_posteriors, epsilon=EPSILON):
@ -366,7 +370,7 @@ class EMQ(AggregativeProbabilisticQuantifier):
# M-step:
qs = ps.mean(axis=0)
if qs_prev_ is not None and qp.error.mae(qs, qs_prev_) < epsilon and s>10:
if qs_prev_ is not None and qp.error.mae(qs, qs_prev_) < epsilon and s > 10:
converged = True
qs_prev_ = qs
@ -389,7 +393,7 @@ class HDy(AggregativeProbabilisticQuantifier, BinaryQuantifier):
self.learner = learner
self.val_split = val_split
def fit(self, data: LabelledCollection, fit_learner=True, val_split: Union[float, LabelledCollection]=None):
def fit(self, data: LabelledCollection, fit_learner=True, val_split: Union[float, LabelledCollection] = None):
"""
Trains a HDy quantifier
:param data: the training set
@ -405,13 +409,15 @@ class HDy(AggregativeProbabilisticQuantifier, BinaryQuantifier):
self._check_binary(data, self.__class__.__name__)
self.learner, validation = training_helper(
self.learner, data, fit_learner, ensure_probabilistic=True, val_split=val_split)
Px = self.posterior_probabilities(validation.instances)[:,1] # takes only the P(y=+1|x)
self.Pxy1 = Px[validation.labels == 1]
self.Pxy0 = Px[validation.labels == 0]
Px = self.posterior_probabilities(validation.instances)[:, 1] # takes only the P(y=+1|x)
self.Pxy1 = Px[validation.labels == self.learner.classes_[1]]
self.Pxy0 = Px[validation.labels == self.learner.classes_[0]]
# pre-compute the histogram for positive and negative examples
self.bins = np.linspace(10, 110, 11, dtype=int) #[10, 20, 30, ..., 100, 110]
self.Pxy1_density = {bins: np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)[0] for bins in self.bins}
self.Pxy0_density = {bins: np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)[0] for bins in self.bins}
self.bins = np.linspace(10, 110, 11, dtype=int) # [10, 20, 30, ..., 100, 110]
self.Pxy1_density = {bins: np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)[0] for bins in
self.bins}
self.Pxy0_density = {bins: np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)[0] for bins in
self.bins}
return self
def aggregate(self, classif_posteriors):
@ -419,12 +425,12 @@ class HDy(AggregativeProbabilisticQuantifier, BinaryQuantifier):
# and the final estimated a priori probability was taken as the median of these 11 estimates."
# (González-Castro, et al., 2013).
Px = classif_posteriors[:,1] # takes only the P(y=+1|x)
Px = classif_posteriors[:, 1] # takes only the P(y=+1|x)
prev_estimations = []
#for bins in np.linspace(10, 110, 11, dtype=int): #[10, 20, 30, ..., 100, 110]
#Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)
#Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)
# for bins in np.linspace(10, 110, 11, dtype=int): #[10, 20, 30, ..., 100, 110]
# Pxy0_density, _ = np.histogram(self.Pxy0, bins=bins, range=(0, 1), density=True)
# Pxy1_density, _ = np.histogram(self.Pxy1, bins=bins, range=(0, 1), density=True)
for bins in self.bins:
Pxy0_density = self.Pxy0_density[bins]
Pxy1_density = self.Pxy1_density[bins]
@ -433,14 +439,14 @@ class HDy(AggregativeProbabilisticQuantifier, BinaryQuantifier):
prev_selected, min_dist = None, None
for prev in F.prevalence_linspace(n_prevalences=100, repeat=1, smooth_limits_epsilon=0.0):
Px_train = prev*Pxy1_density + (1 - prev)*Pxy0_density
Px_train = prev * Pxy1_density + (1 - prev) * Pxy0_density
hdy = F.HellingerDistance(Px_train, Px_test)
if prev_selected is None or hdy < min_dist:
prev_selected, min_dist = prev, hdy
prev_estimations.append(prev_selected)
pos_class_prev = np.median(prev_estimations)
return np.asarray([1-pos_class_prev, pos_class_prev])
class1_prev = np.median(prev_estimations)
return np.asarray([1 - class1_prev, class1_prev])
class ELM(AggregativeQuantifier, BinaryQuantifier):
@ -457,8 +463,8 @@ class ELM(AggregativeQuantifier, BinaryQuantifier):
self.learner.fit(data.instances, data.labels)
return self
def aggregate(self, classif_predictions:np.ndarray):
return F.prevalence_from_labels(classif_predictions, self.learner.n_classes_)
def aggregate(self, classif_predictions: np.ndarray):
return F.prevalence_from_labels(classif_predictions, self.classes_)
def classify(self, X, y=None):
return self.learner.predict(X)
@ -470,6 +476,7 @@ class SVMQ(ELM):
Quantification-oriented learning based on reliable classifiers.
Pattern Recognition, 48(2):591604.
"""
def __init__(self, svmperf_base=None, **kwargs):
super(SVMQ, self).__init__(svmperf_base, loss='q', **kwargs)
@ -480,6 +487,7 @@ class SVMKLD(ELM):
Optimizing text quantifiers for multivariate loss functions.
ACM Transactions on Knowledge Discovery and Data, 9(4):Article 27.
"""
def __init__(self, svmperf_base=None, **kwargs):
super(SVMKLD, self).__init__(svmperf_base, loss='kld', **kwargs)
@ -490,6 +498,7 @@ class SVMNKLD(ELM):
Optimizing text quantifiers for multivariate loss functions.
ACM Transactions on Knowledge Discovery and Data, 9(4):Article 27.
"""
def __init__(self, svmperf_base=None, **kwargs):
super(SVMNKLD, self).__init__(svmperf_base, loss='nkld', **kwargs)
@ -531,7 +540,7 @@ class OneVsAll(AggregativeQuantifier):
f'{self.__class__.__name__} expect non-binary data'
assert isinstance(self.binary_quantifier, BaseQuantifier), \
f'{self.binary_quantifier} does not seem to be a Quantifier'
assert fit_learner==True, 'fit_learner must be True'
assert fit_learner == True, 'fit_learner must be True'
self.dict_binary_quantifiers = {c: deepcopy(self.binary_quantifier) for c in data.classes_}
self.__parallel(self._delayed_binary_fit, data)
@ -559,11 +568,11 @@ class OneVsAll(AggregativeQuantifier):
def aggregate(self, classif_predictions_bin):
if self.probabilistic:
assert classif_predictions_bin.shape[1]==self.n_classes and classif_predictions_bin.shape[2]==2, \
assert classif_predictions_bin.shape[1] == self.n_classes and classif_predictions_bin.shape[2] == 2, \
'param classif_predictions_bin does not seem to be a valid matrix (ndarray) of posterior ' \
'probabilities (2 dimensions) for each document (row) and class (columns)'
else:
assert set(np.unique(classif_predictions_bin)).issubset({0,1}), \
assert set(np.unique(classif_predictions_bin)).issubset({0, 1}), \
'param classif_predictions_bin does not seem to be a valid matrix (ndarray) of binary ' \
'predictions for each document (row) and class (columns)'
prevalences = self.__parallel(self._delayed_binary_aggregate, classif_predictions_bin)
@ -606,7 +615,7 @@ class OneVsAll(AggregativeQuantifier):
return self.dict_binary_quantifiers[c].aggregate(classif_predictions[:, c])[1]
def _delayed_binary_fit(self, c, data):
bindata = LabelledCollection(data.instances, data.labels == c, n_classes=2)
bindata = LabelledCollection(data.instances, data.labels == c, classes_=[False, True])
self.dict_binary_quantifiers[c].fit(bindata)
@property
@ -616,9 +625,3 @@ class OneVsAll(AggregativeQuantifier):
@property
def probabilistic(self):
return self.binary_quantifier.probabilistic

View File

@ -19,6 +19,10 @@ class BaseQuantifier(metaclass=ABCMeta):
@abstractmethod
def get_params(self, deep=True): ...
@property
@abstractmethod
def classes_(self): ...
# these methods allows meta-learners to reimplement the decision based on their constituents, and not
# based on class structure
@property

View File

@ -186,6 +186,10 @@ class Ensemble(BaseQuantifier):
order = np.argsort(dist)
return _select_k(predictions, order, k=self.red_size)
@property
def classes_(self):
return self.base_quantifier.classes_
@property
def binary(self):
return self.base_quantifier.binary

View File

@ -58,6 +58,7 @@ class QuaNetTrainer(BaseQuantifier):
self.device = torch.device(device)
self.__check_params_colision(self.quanet_params, self.learner.get_params())
self._classes_ = None
def fit(self, data: LabelledCollection, fit_learner=True):
"""
@ -67,6 +68,7 @@ class QuaNetTrainer(BaseQuantifier):
:param fit_learner: if true, trains the classifier on a split containing 40% of the data
:return: self
"""
self._classes_ = data.classes_
classifier_data, unused_data = data.split_stratified(0.4)
train_data, valid_data = unused_data.split_stratified(0.66) # 0.66 split of 60% makes 40% and 20%
@ -256,6 +258,10 @@ class QuaNetTrainer(BaseQuantifier):
import shutil
shutil.rmtree(self.checkpointdir, ignore_errors=True)
@property
def classes_(self):
return self._classes_
def mae_loss(output, target):
return torch.mean(torch.abs(output - target))

View File

@ -2,18 +2,22 @@ from quapy.data import LabelledCollection
from .base import BaseQuantifier
class MaximumLikelihoodPrevalenceEstimation(BaseQuantifier):
def __init__(self, **kwargs):
pass
self._classes_ = None
def fit(self, data: LabelledCollection, *args):
self._classes_ = data.classes_
self.estimated_prevalence = data.prevalence()
def quantify(self, documents, *args):
return self.estimated_prevalence
@property
def classes_(self):
return self._classes_
def get_params(self):
pass

View File

@ -4,7 +4,6 @@ from copy import deepcopy
from typing import Union, Callable
import quapy as qp
import quapy.functional as F
from quapy.data.base import LabelledCollection
from quapy.evaluation import artificial_sampling_prediction
from quapy.method.aggregative import BaseQuantifier
@ -80,7 +79,7 @@ class GridSearchQ(BaseQuantifier):
return training, validation
elif isinstance(validation, float):
assert 0. < validation < 1., 'validation proportion should be in (0,1)'
training, validation = training.split_stratified(train_prop=1-validation)
training, validation = training.split_stratified(train_prop=1 - validation)
return training, validation
else:
raise ValueError(f'"validation" must either be a LabelledCollection or a float in (0,1) indicating the'
@ -97,7 +96,7 @@ class GridSearchQ(BaseQuantifier):
raise ValueError(f'unexpected error type; must either be a callable function or a str representing\n'
f'the name of an error function in {qp.error.QUANTIFICATION_ERROR_NAMES}')
def fit(self, training: LabelledCollection, val_split: Union[LabelledCollection, float]=None):
def fit(self, training: LabelledCollection, val_split: Union[LabelledCollection, float] = None):
"""
:param training: the training set on which to optimize the hyperparameters
:param val_split: either a LabelledCollection on which to test the performance of the different settings, or
@ -118,6 +117,7 @@ class GridSearchQ(BaseQuantifier):
def handler(signum, frame):
self.sout('timeout reached')
raise TimeoutError()
signal.signal(signal.SIGALRM, handler)
self.sout(f'starting optimization with n_jobs={n_jobs}')
@ -175,6 +175,10 @@ class GridSearchQ(BaseQuantifier):
def quantify(self, instances):
return self.best_model_.quantify(instances)
@property
def classes_(self):
return self.best_model_.classes_
def set_params(self, **parameters):
self.param_grid = parameters
@ -185,4 +189,3 @@ class GridSearchQ(BaseQuantifier):
if hasattr(self, 'best_model_'):
return self.best_model_
raise ValueError('best_model called before fit')

View File

@ -6,13 +6,38 @@ from quapy.data.datasets import REVIEWS_SENTIMENT_DATASETS, TWITTER_SENTIMENT_DA
@pytest.mark.parametrize('dataset_name', REVIEWS_SENTIMENT_DATASETS)
def test_fetch_reviews(dataset_name):
fetch_reviews(dataset_name)
dataset = fetch_reviews(dataset_name)
print(f'Dataset {dataset_name}')
print('Training set stats')
dataset.training.stats()
print('Test set stats')
dataset.test.stats()
@pytest.mark.parametrize('dataset_name', TWITTER_SENTIMENT_DATASETS_TEST + TWITTER_SENTIMENT_DATASETS_TRAIN)
def test_fetch_twitter(dataset_name):
fetch_twitter(dataset_name)
try:
dataset = fetch_twitter(dataset_name)
except ValueError as ve:
if dataset_name == 'semeval' and ve.args[0].startswith(
'dataset "semeval" can only be used for model selection.'):
dataset = fetch_twitter(dataset_name, for_model_selection=True)
print(f'Dataset {dataset_name}')
print('Training set stats')
dataset.training.stats()
print('Test set stats')
@pytest.mark.parametrize('dataset_name', UCI_DATASETS)
@pytest.mark.parametrize('dataset_name', UCI_DATASETS)
def test_fetch_UCIDataset(dataset_name):
fetch_UCIDataset(dataset_name)
try:
dataset = fetch_UCIDataset(dataset_name)
except FileNotFoundError as fnfe:
if dataset_name == 'pageblocks.5' and fnfe.args[0].find(
'If this is the first time you attempt to load this dataset') > 0:
print('The pageblocks.5 dataset requires some hand processing to be usable, skipping this test.')
return
print(f'Dataset {dataset_name}')
print('Training set stats')
dataset.training.stats()
print('Test set stats')

View File

@ -1,24 +1,30 @@
import numpy
import pytest
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
import quapy as qp
from quapy.data import Dataset, LabelledCollection
from quapy.method import AGGREGATIVE_METHODS, NON_AGGREGATIVE_METHODS, EXPLICIT_LOSS_MINIMIZATION_METHODS
from quapy.method.aggregative import ACC, PACC, HDy
from quapy.method.meta import Ensemble
datasets = [qp.datasets.fetch_twitter('semeval16')]
datasets = [pytest.param(qp.datasets.fetch_twitter('hcr'), id='hcr'),
pytest.param(qp.datasets.fetch_UCIDataset('ionosphere'), id='ionosphere')]
aggregative_methods = [qp.method.aggregative.CC, qp.method.aggregative.ACC, qp.method.aggregative.ELM]
learners = [LogisticRegression, MultinomialNB, LinearSVC]
learners = [LogisticRegression, LinearSVC]
@pytest.mark.parametrize('dataset', datasets)
@pytest.mark.parametrize('aggregative_method', aggregative_methods)
@pytest.mark.parametrize('aggregative_method', AGGREGATIVE_METHODS.difference(EXPLICIT_LOSS_MINIMIZATION_METHODS))
@pytest.mark.parametrize('learner', learners)
def test_aggregative_methods(dataset, aggregative_method, learner):
def test_aggregative_methods(dataset: Dataset, aggregative_method, learner):
model = aggregative_method(learner())
if model.binary and not dataset.binary:
print(f'skipping the test of binary model {type(model)} on non-binary dataset {dataset}')
return
model.fit(dataset.training)
estim_prevalences = model.quantify(dataset.test.instances)
@ -27,3 +33,147 @@ def test_aggregative_methods(dataset, aggregative_method, learner):
error = qp.error.mae(true_prevalences, estim_prevalences)
assert type(error) == numpy.float64
@pytest.mark.parametrize('dataset', datasets)
@pytest.mark.parametrize('elm_method', EXPLICIT_LOSS_MINIMIZATION_METHODS)
def test_elm_methods(dataset: Dataset, elm_method):
try:
model = elm_method()
except AssertionError as ae:
if ae.args[0].find('does not seem to point to a valid path') > 0:
print('Missing SVMperf binary program, skipping test')
return
if model.binary and not dataset.binary:
print(f'skipping the test of binary model {model} on non-binary dataset {dataset}')
return
model.fit(dataset.training)
estim_prevalences = model.quantify(dataset.test.instances)
true_prevalences = dataset.test.prevalence()
error = qp.error.mae(true_prevalences, estim_prevalences)
assert type(error) == numpy.float64
@pytest.mark.parametrize('dataset', datasets)
@pytest.mark.parametrize('non_aggregative_method', NON_AGGREGATIVE_METHODS)
def test_non_aggregative_methods(dataset: Dataset, non_aggregative_method):
model = non_aggregative_method()
if model.binary and not dataset.binary:
print(f'skipping the test of binary model {model} on non-binary dataset {dataset}')
return
model.fit(dataset.training)
estim_prevalences = model.quantify(dataset.test.instances)
true_prevalences = dataset.test.prevalence()
error = qp.error.mae(true_prevalences, estim_prevalences)
assert type(error) == numpy.float64
@pytest.mark.parametrize('base_method', AGGREGATIVE_METHODS.difference(EXPLICIT_LOSS_MINIMIZATION_METHODS))
@pytest.mark.parametrize('learner', learners)
@pytest.mark.parametrize('dataset', datasets)
@pytest.mark.parametrize('policy', Ensemble.VALID_POLICIES)
def test_ensemble_method(base_method, learner, dataset: Dataset, policy):
qp.environ['SAMPLE_SIZE'] = len(dataset.training)
model = Ensemble(quantifier=base_method(learner()), size=5, policy=policy, n_jobs=-1)
if model.binary and not dataset.binary:
print(f'skipping the test of binary model {model} on non-binary dataset {dataset}')
return
model.fit(dataset.training)
estim_prevalences = model.quantify(dataset.test.instances)
true_prevalences = dataset.test.prevalence()
error = qp.error.mae(true_prevalences, estim_prevalences)
assert type(error) == numpy.float64
def test_quanet_method():
dataset = qp.datasets.fetch_reviews('kindle', pickle=True)
dataset = Dataset(dataset.training.sampling(100, *dataset.training.prevalence()),
dataset.test.sampling(100, *dataset.test.prevalence()))
qp.data.preprocessing.index(dataset, min_df=5, inplace=True)
from quapy.classification.neural import CNNnet
cnn = CNNnet(dataset.vocabulary_size, dataset.training.n_classes)
from quapy.classification.neural import NeuralClassifierTrainer
learner = NeuralClassifierTrainer(cnn, device='cuda')
from quapy.method.meta import QuaNet
model = QuaNet(learner, sample_size=len(dataset.training), device='cuda')
if model.binary and not dataset.binary:
print(f'skipping the test of binary model {model} on non-binary dataset {dataset}')
return
model.fit(dataset.training)
estim_prevalences = model.quantify(dataset.test.instances)
true_prevalences = dataset.test.prevalence()
error = qp.error.mae(true_prevalences, estim_prevalences)
assert type(error) == numpy.float64
def models_to_test_for_str_label_names():
models = list()
learner = LogisticRegression
for method in AGGREGATIVE_METHODS.difference(EXPLICIT_LOSS_MINIMIZATION_METHODS):
models.append(method(learner()))
for method in NON_AGGREGATIVE_METHODS:
models.append(method())
return models
@pytest.mark.parametrize('model', models_to_test_for_str_label_names())
def test_str_label_names(model):
if type(model) in {ACC, PACC, HDy}:
print(
f'skipping the test of binary model {type(model)} because it currently does not support random seed control.')
return
dataset = qp.datasets.fetch_reviews('imdb', pickle=True)
dataset = Dataset(dataset.training.sampling(1000, *dataset.training.prevalence()),
dataset.test.sampling(1000, *dataset.test.prevalence()))
qp.data.preprocessing.text2tfidf(dataset, min_df=5, inplace=True)
model.fit(dataset.training)
int_estim_prevalences = model.quantify(dataset.test.instances)
true_prevalences = dataset.test.prevalence()
error = qp.error.mae(true_prevalences, int_estim_prevalences)
assert type(error) == numpy.float64
dataset_str = Dataset(LabelledCollection(dataset.training.instances,
['one' if label == 1 else 'zero' for label in dataset.training.labels]),
LabelledCollection(dataset.test.instances,
['one' if label == 1 else 'zero' for label in dataset.test.labels]))
model.fit(dataset_str.training)
str_estim_prevalences = model.quantify(dataset_str.test.instances)
true_prevalences = dataset_str.test.prevalence()
error = qp.error.mae(true_prevalences, str_estim_prevalences)
assert type(error) == numpy.float64
print(true_prevalences)
print(int_estim_prevalences)
print(str_estim_prevalences)
numpy.testing.assert_almost_equal(int_estim_prevalences[1],
str_estim_prevalences[list(model.classes_).index('one')])