Fix 10 correctness bugs and clean up warnings/logging in core library
Correctness: - OneVsAllAggregative.aggregation_fit: fix undefined variable and call to the nonexistent aggregate_fit (should be aggregation_fit) - solve_adjustment: stop mutating the caller's fitted arrays in place for method='invariant-ratio' - LabelledCollection.join(): fix classes always being None due to ndarray.sort() returning None; now unions each collection's own classes_ so a class absent from a particular join is kept at zero prevalence - Rename the duplicate newSVMKLD (nkld variant) to newSVMNKLD so both loss variants are reachable - EMQ/DyS/DMy: resolve n_jobs via qp._get_njobs() like the other methods, so qp.environ['N_JOBS'] is respected - AggregativeMedianEstimator: drop backend='threading' (global np.random state mutated via temp_seed is not thread-safe); use the safe process based default instead - NeuralClassifier: default device now 'cpu', matching its own docstring - ConfidenceEllipseSimplex: narrow bare except to np.linalg.LinAlgError - SVMperf: stop merging stderr into stdout so failures report the actual subprocess error instead of crashing with AttributeError - ConfidenceRegionABC: replace @lru_cache on bound methods (leaked every instance for the process lifetime) with per-instance caching Style/quality: - Replace print() with warnings.warn()/logging across aggregative.py, base.py, meta.py, model_selection.py, classification/neural.py, method/_neural.py, classification/svmperf.py, data/reader.py, data/datasets.py; also fixes a `raise RuntimeWarning(...)` in EMQ that would have crashed instead of warning - Remove dead duplicate class MedianEstimator2 in meta.py - Rename misleading _compute_tpr(TP, FP) parameter to FN, matching what callers actually pass - Replace argparse.ArgumentError misuse with ValueError - Remove commented-out dead code in protocol.py - _lequa.py: fix CSV-parse failure raising an unrelated UnboundLocalError instead of a clear ValueError Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
f6c822ca8f
commit
b29966797a
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
import os
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from pathlib import Path
|
||||
|
|
@ -42,7 +43,7 @@ class NeuralClassifierTrainer:
|
|||
batch_size=64,
|
||||
batch_size_test=512,
|
||||
padding_length=300,
|
||||
device='cuda',
|
||||
device='cpu',
|
||||
checkpointpath='../checkpoint/classifier_net.dat'):
|
||||
|
||||
super().__init__()
|
||||
|
|
@ -63,7 +64,7 @@ class NeuralClassifierTrainer:
|
|||
self.learner_hyperparams = self.net.get_params()
|
||||
self.checkpointpath = checkpointpath
|
||||
|
||||
print(f'[NeuralNetwork running on {device}]')
|
||||
logging.getLogger(__name__).info(f'NeuralNetwork running on {device}')
|
||||
os.makedirs(Path(checkpointpath).parent, exist_ok=True)
|
||||
|
||||
def reset_net_params(self, vocab_size, n_classes):
|
||||
|
|
@ -198,14 +199,15 @@ class NeuralClassifierTrainer:
|
|||
if self.early_stop.IMPROVED:
|
||||
torch.save(self.net.state_dict(), checkpoint)
|
||||
elif self.early_stop.STOP:
|
||||
print(f'training ended by patience exhasted; loading best model parameters in {checkpoint} '
|
||||
f'for epoch {self.early_stop.best_epoch}')
|
||||
logging.getLogger(__name__).info(
|
||||
f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
|
||||
f'for epoch {self.early_stop.best_epoch}')
|
||||
self.net.load_state_dict(torch.load(checkpoint))
|
||||
break
|
||||
|
||||
print('performing one training pass over the validation set...')
|
||||
logging.getLogger(__name__).info('performing one training pass over the validation set...')
|
||||
self._train_epoch(valid_generator, self.status['tr'], pbar, epoch=0)
|
||||
print('[done]')
|
||||
logging.getLogger(__name__).info('done')
|
||||
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
|
|
@ -79,14 +80,14 @@ class SVMperf(BaseEstimator, ClassifierMixin):
|
|||
|
||||
cmd = ' '.join([self.svmperf_learn, self.c_cmd, self.loss_cmd, traindat, self.model])
|
||||
if self.verbose:
|
||||
print('[Running]', cmd)
|
||||
p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT)
|
||||
logging.getLogger(__name__).info(f'[Running] {cmd}')
|
||||
p = subprocess.run(cmd.split(), stdout=PIPE, stderr=PIPE)
|
||||
if not exists(self.model):
|
||||
print(p.stderr.decode('utf-8'))
|
||||
logging.getLogger(__name__).error(p.stderr.decode('utf-8'))
|
||||
remove(traindat)
|
||||
|
||||
if self.verbose:
|
||||
print(p.stdout.decode('utf-8'))
|
||||
logging.getLogger(__name__).info(p.stdout.decode('utf-8'))
|
||||
|
||||
return self
|
||||
|
||||
|
|
@ -125,11 +126,11 @@ class SVMperf(BaseEstimator, ClassifierMixin):
|
|||
|
||||
cmd = ' '.join([self.svmperf_classify, testdat, self.model, predictions_path])
|
||||
if self.verbose:
|
||||
print('[Running]', cmd)
|
||||
logging.getLogger(__name__).info(f'[Running] {cmd}')
|
||||
p = subprocess.run(cmd.split(), stdout=PIPE, stderr=STDOUT)
|
||||
|
||||
if self.verbose:
|
||||
print(p.stdout.decode('utf-8'))
|
||||
logging.getLogger(__name__).info(p.stdout.decode('utf-8'))
|
||||
|
||||
scores = np.loadtxt(predictions_path)
|
||||
remove(testdat)
|
||||
|
|
|
|||
|
|
@ -187,8 +187,7 @@ class ResultSubmission:
|
|||
try:
|
||||
df = pd.read_csv(path, index_col=0)
|
||||
except Exception as e:
|
||||
print(f'the file {path} does not seem to be a valid csv file. ')
|
||||
print(e)
|
||||
raise ValueError(f'the file {path} does not seem to be a valid csv file: {e}')
|
||||
return ResultSubmission.check_dataframe_format(df, path=path)
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -329,7 +329,9 @@ class LabelledCollection:
|
|||
else:
|
||||
raise NotImplementedError('unsupported operation for collection types')
|
||||
labels = np.concatenate([lc.labels for lc in args])
|
||||
classes = np.unique(labels).sort()
|
||||
# union of each collection's own classes_, so a class declared but absent from
|
||||
# this particular join (e.g. an empty fold) is preserved at zero prevalence
|
||||
classes = np.unique(np.concatenate([lc.classes_ for lc in args]))
|
||||
return LabelledCollection(instances, labels, classes=classes)
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
import zipfile
|
||||
|
|
@ -212,8 +213,9 @@ def fetch_twitter(dataset_name, for_model_selection=False, min_df=None, data_hom
|
|||
if dataset_name in {'semeval13', 'semeval14', 'semeval15'}:
|
||||
trainset_name = 'semeval'
|
||||
testset_name = 'semeval' if for_model_selection else dataset_name
|
||||
print(f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common "
|
||||
f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}")
|
||||
logging.getLogger(__name__).info(
|
||||
f"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common "
|
||||
f"(called 'semeval'); returning trainin-set='{trainset_name}' and test-set={testset_name}")
|
||||
else:
|
||||
if dataset_name == 'semeval' and for_model_selection==False:
|
||||
raise ValueError('dataset "semeval" can only be used for model selection. '
|
||||
|
|
@ -1152,11 +1154,3 @@ def fetch_image_embeddings(dataset_name, embedding, heldout_only=True, data_home
|
|||
return Dataset(train, test, name=dataset_name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
#train, val, test = _fetch_image_embedding_splits(dataset_name='mnist', embedding='logits')
|
||||
#print(train)
|
||||
#print(val)
|
||||
#print(test)
|
||||
|
||||
dataset = fetch_image_embeddings(dataset_name='svhn', embedding='features', heldout_only=True)
|
||||
print(dataset)
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
import logging
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import dok_matrix
|
||||
from tqdm import tqdm
|
||||
|
|
@ -30,7 +32,7 @@ def from_text(path, encoding='utf-8', verbose=1, class2int=True):
|
|||
all_sentences.append(sentence)
|
||||
all_labels.append(label)
|
||||
except ValueError:
|
||||
print(f'format error in {line}')
|
||||
logging.getLogger(__name__).warning(f'format error in {line}')
|
||||
return all_sentences, all_labels
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -650,7 +650,11 @@ def solve_adjustment(
|
|||
if method == "inversion":
|
||||
pass # We leave A and B unchanged
|
||||
elif method == "invariant-ratio":
|
||||
# Change the last equation to replace it with the normalization condition
|
||||
# Change the last equation to replace it with the normalization condition;
|
||||
# copy first so this does not mutate the caller's arrays (np.asarray above
|
||||
# returns the same object, not a copy, when the input is already float64)
|
||||
A = A.copy()
|
||||
B = B.copy()
|
||||
A[-1, :] = 1.0
|
||||
B[-1] = 1.0
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
|
|
@ -173,7 +174,7 @@ class QuaNetTrainer(BaseQuantifier):
|
|||
order_by=0 if data.binary else None,
|
||||
**self.quanet_params
|
||||
).to(self.device)
|
||||
print(self.quanet)
|
||||
logging.getLogger(__name__).debug(self.quanet)
|
||||
|
||||
self.optim = torch.optim.Adam(self.quanet.parameters(), lr=self.lr)
|
||||
early_stop = EarlyStop(self.patience, lower_is_better=True)
|
||||
|
|
@ -188,8 +189,9 @@ class QuaNetTrainer(BaseQuantifier):
|
|||
if early_stop.IMPROVED:
|
||||
torch.save(self.quanet.state_dict(), checkpoint)
|
||||
elif early_stop.STOP:
|
||||
print(f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
|
||||
f'for epoch {early_stop.best_epoch}')
|
||||
logging.getLogger(__name__).info(
|
||||
f'training ended by patience exhausted; loading best model parameters in {checkpoint} '
|
||||
f'for epoch {early_stop.best_epoch}')
|
||||
self.quanet.load_state_dict(torch.load(checkpoint))
|
||||
break
|
||||
|
||||
|
|
|
|||
|
|
@ -110,10 +110,10 @@ class ThresholdOptimization(BinaryAggregativeQuantifier):
|
|||
TN = np.logical_and(y == y_, y == self.neg_label).sum()
|
||||
return TP, FP, FN, TN
|
||||
|
||||
def _compute_tpr(self, TP, FP):
|
||||
if TP + FP == 0:
|
||||
def _compute_tpr(self, TP, FN):
|
||||
if TP + FN == 0:
|
||||
return 1
|
||||
return TP / (TP + FP)
|
||||
return TP / (TP + FN)
|
||||
|
||||
def _compute_fpr(self, FP, TN):
|
||||
if FP + TN == 0:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from argparse import ArgumentError
|
||||
from copy import deepcopy
|
||||
from typing import Callable, Literal, Union
|
||||
import numpy as np
|
||||
|
|
@ -82,9 +82,9 @@ class AggregativeQuantifier(BaseQuantifier, ABC):
|
|||
(f'when {val_split=} is indicated as an integer, it represents the number of folds in a kFCV '
|
||||
f'and must thus be >1')
|
||||
if val_split==5 and not fit_classifier:
|
||||
print(f'Warning: {val_split=} will be ignored when the classifier is already trained '
|
||||
f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} '
|
||||
f'to None to avoid this warning.')
|
||||
warnings.warn(f'{val_split=} will be ignored when the classifier is already trained '
|
||||
f'({fit_classifier=}). Parameter {self.val_split=} will be set to None. Set {val_split=} '
|
||||
f'to None to avoid this warning.')
|
||||
self.val_split=None
|
||||
if val_split!=5:
|
||||
assert fit_classifier, (f'Parameter {val_split=} has been modified, but {fit_classifier=} '
|
||||
|
|
@ -343,8 +343,8 @@ class AggregativeSoftQuantifier(AggregativeQuantifier, ABC):
|
|||
"""
|
||||
if not hasattr(self.classifier, self._classifier_method()):
|
||||
if adapt_if_necessary:
|
||||
print(f'warning: The learner {self.classifier.__class__.__name__} does not seem to be '
|
||||
f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).')
|
||||
warnings.warn(f'The learner {self.classifier.__class__.__name__} does not seem to be '
|
||||
f'probabilistic. The learner will be calibrated (using CalibratedClassifierCV).')
|
||||
self.classifier = CalibratedClassifierCV(self.classifier, cv=5)
|
||||
else:
|
||||
raise AssertionError(f'error: The learner {self.classifier.__class__.__name__} does not '
|
||||
|
|
@ -838,7 +838,7 @@ class EMQ(AggregativeSoftQuantifier):
|
|||
self.exact_train_prev = exact_train_prev
|
||||
self.calib = calib
|
||||
self.on_calib_error = on_calib_error
|
||||
self.n_jobs = n_jobs
|
||||
self.n_jobs = qp._get_njobs(n_jobs)
|
||||
|
||||
@classmethod
|
||||
def EMQ_BCTS(cls, classifier: BaseEstimator, fit_classifier=True, val_split=5, on_calib_error="raise", n_jobs=None):
|
||||
|
|
@ -875,15 +875,15 @@ class EMQ(AggregativeSoftQuantifier):
|
|||
def _check_init_parameters(self):
|
||||
if self.val_split is not None:
|
||||
if self.exact_train_prev and self.calib is None:
|
||||
raise RuntimeWarning(f'The parameter {self.val_split=} was specified for EMQ, while the parameters '
|
||||
f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an '
|
||||
f'unnecessary overload.')
|
||||
warnings.warn(f'The parameter {self.val_split=} was specified for EMQ, while the parameters '
|
||||
f'{self.exact_train_prev=} and {self.calib=}. This has no effect and causes an '
|
||||
f'unnecessary overload.', RuntimeWarning)
|
||||
else:
|
||||
if self.calib is not None:
|
||||
print(f'[warning] The parameter {self.calib=} requires the val_split be different from None. '
|
||||
f'This parameter will be set to 5. To avoid this warning, set this value to a float value '
|
||||
f'indicating the proportion of training data to be used as validation, or to an integer '
|
||||
f'indicating the number of folds for kFCV.')
|
||||
warnings.warn(f'The parameter {self.calib=} requires the val_split be different from None. '
|
||||
f'This parameter will be set to 5. To avoid this warning, set this value to a float value '
|
||||
f'indicating the proportion of training data to be used as validation, or to an integer '
|
||||
f'indicating the number of folds for kFCV.')
|
||||
self.val_split = 5
|
||||
|
||||
def classify(self, X):
|
||||
|
|
@ -945,14 +945,14 @@ class EMQ(AggregativeSoftQuantifier):
|
|||
requires_predictions = (self.calib is not None) or (not self.exact_train_prev)
|
||||
if P is None and requires_predictions:
|
||||
# classifier predictions were not generated because val_split=None
|
||||
raise ArgumentError(self.val_split, self.__class__.__name__ +
|
||||
": Classifier predictions for the aggregative fit were not generated because "
|
||||
"val_split=None. This usually happens when you enable calibrations or heuristics "
|
||||
"during model selection but left val_split set to its default value (None). "
|
||||
"Please provide one of the following values for val_split: (i) an integer >1 "
|
||||
"(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. "
|
||||
"val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit "
|
||||
"validation data")
|
||||
raise ValueError(self.__class__.__name__ +
|
||||
": Classifier predictions for the aggregative fit were not generated because "
|
||||
"val_split=None. This usually happens when you enable calibrations or heuristics "
|
||||
"during model selection but left val_split set to its default value (None). "
|
||||
"Please provide one of the following values for val_split: (i) an integer >1 "
|
||||
"(e.g. val_split=5) for k-fold cross-validation; (ii) a float in (0,1) (e.g. "
|
||||
"val_split=0.3) for a proportion split; or (iii) a tuple (X, y) with explicit "
|
||||
"validation data")
|
||||
|
||||
if self.calib is not None:
|
||||
calibrator = _get_abstention_calibrators().get(self.calib, None)
|
||||
|
|
@ -1023,7 +1023,7 @@ class EMQ(AggregativeSoftQuantifier):
|
|||
s += 1
|
||||
|
||||
if not converged:
|
||||
print('[warning] the method has reached the maximum number of iterations; it might have not converged')
|
||||
warnings.warn('the method has reached the maximum number of iterations; it might have not converged')
|
||||
|
||||
return qs, ps
|
||||
|
||||
|
|
@ -1143,7 +1143,7 @@ class DyS(AggregativeSoftQuantifier, BinaryAggregativeQuantifier):
|
|||
self.tol = tol
|
||||
self.divergence = divergence
|
||||
self.n_bins = n_bins
|
||||
self.n_jobs = n_jobs
|
||||
self.n_jobs = qp._get_njobs(n_jobs)
|
||||
|
||||
def _ternary_search(self, f, left, right, tol):
|
||||
"""
|
||||
|
|
@ -1275,7 +1275,7 @@ class DMy(AggregativeSoftQuantifier):
|
|||
self.divergence = divergence
|
||||
self.cdf = cdf
|
||||
self.search = search
|
||||
self.n_jobs = n_jobs
|
||||
self.n_jobs = qp._get_njobs(n_jobs)
|
||||
|
||||
@classmethod
|
||||
def HDy(cls, classifier: BaseEstimator = None, fit_classifier=True, val_split=5, n_jobs=None):
|
||||
|
|
@ -1449,9 +1449,9 @@ def newSVMKLD(svmperf_base=None, C=1):
|
|||
return newELM(svmperf_base, loss='kld', C=C)
|
||||
|
||||
|
||||
def newSVMKLD(svmperf_base=None, C=1):
|
||||
def newSVMNKLD(svmperf_base=None, C=1):
|
||||
"""
|
||||
SVM(KLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
|
||||
SVM(NKLD) is an Explicit Loss Minimization (ELM) quantifier set to optimize for the Kullback-Leibler Divergence
|
||||
normalized via the logistic function, as proposed by
|
||||
`Esuli et al. 2015 <https://dl.acm.org/doi/abs/10.1145/2700406>`_.
|
||||
Equivalent to:
|
||||
|
|
@ -1578,7 +1578,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
|
|||
return F.normalize_prevalence(prevalences)
|
||||
|
||||
def aggregation_fit(self, classif_predictions, labels):
|
||||
self._parallel(self._delayed_binary_aggregate_fit(c, classif_predictions, labels))
|
||||
self._parallel(self._delayed_binary_aggregate_fit, classif_predictions, labels)
|
||||
return self
|
||||
|
||||
def _delayed_binary_classification(self, c, X):
|
||||
|
|
@ -1590,7 +1590,7 @@ class OneVsAllAggregative(OneVsAllGeneric, AggregativeQuantifier):
|
|||
|
||||
def _delayed_binary_aggregate_fit(self, c, classif_predictions, labels):
|
||||
# trains the aggregation function of the cth quantifier
|
||||
return self.dict_binary_quantifiers[c].aggregate_fit(classif_predictions[:, c], labels)
|
||||
return self.dict_binary_quantifiers[c].aggregation_fit(classif_predictions[:, c], labels == c)
|
||||
|
||||
|
||||
class AggregativeMedianEstimator(BinaryQuantifier):
|
||||
|
|
@ -1656,8 +1656,7 @@ class AggregativeMedianEstimator(BinaryQuantifier):
|
|||
((params, X, y) for params in cls_configs),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs,
|
||||
asarray=False,
|
||||
backend='threading'
|
||||
asarray=False
|
||||
)
|
||||
else:
|
||||
model = self.base_quantifier
|
||||
|
|
@ -1669,8 +1668,7 @@ class AggregativeMedianEstimator(BinaryQuantifier):
|
|||
self._delayed_fit_aggregation,
|
||||
itertools.product(models_preds, q_configs),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs,
|
||||
backend='threading'
|
||||
n_jobs=self.n_jobs
|
||||
)
|
||||
else:
|
||||
configs = qp.model_selection.expand_grid(self.param_grid)
|
||||
|
|
@ -1678,8 +1676,7 @@ class AggregativeMedianEstimator(BinaryQuantifier):
|
|||
self._delayed_fit,
|
||||
((params, X, y) for params in configs),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs,
|
||||
backend='threading'
|
||||
n_jobs=self.n_jobs
|
||||
)
|
||||
return self
|
||||
|
||||
|
|
@ -1692,8 +1689,7 @@ class AggregativeMedianEstimator(BinaryQuantifier):
|
|||
self._delayed_predict,
|
||||
((model, instances) for model in self.models),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs,
|
||||
backend='threading'
|
||||
n_jobs=self.n_jobs
|
||||
)
|
||||
return np.median(prev_preds, axis=0)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import warnings
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from copy import deepcopy
|
||||
|
||||
|
|
@ -84,8 +85,8 @@ class OneVsAllGeneric(OneVsAll, BaseQuantifier):
|
|||
assert isinstance(binary_quantifier, BaseQuantifier), \
|
||||
f'{binary_quantifier} does not seem to be a Quantifier'
|
||||
if isinstance(binary_quantifier, qp.method.aggregative.AggregativeQuantifier):
|
||||
print('[warning] the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; '
|
||||
f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}')
|
||||
warnings.warn('the quantifier seems to be an instance of qp.method.aggregative.AggregativeQuantifier; '
|
||||
f'you might prefer instantiating {qp.method.aggregative.OneVsAllAggregative.__name__}')
|
||||
self.binary_quantifier = binary_quantifier
|
||||
self.n_jobs = qp._get_njobs(n_jobs)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from sklearn.utils import resample
|
|||
from abc import ABC, abstractmethod
|
||||
from scipy.special import factorial
|
||||
import copy
|
||||
from functools import lru_cache
|
||||
from tqdm import tqdm
|
||||
|
||||
"""
|
||||
|
|
@ -64,7 +63,6 @@ class ConfidenceRegionABC(ABC):
|
|||
"""
|
||||
...
|
||||
|
||||
@lru_cache
|
||||
def simplex_portion(self):
|
||||
"""
|
||||
Computes the fraction of the simplex which is covered by the region. This is not the volume of the region
|
||||
|
|
@ -73,9 +71,10 @@ class ConfidenceRegionABC(ABC):
|
|||
|
||||
:return: float, the fraction of the simplex covered by the region
|
||||
"""
|
||||
return self.montecarlo_proportion()
|
||||
if not hasattr(self, '_simplex_portion_cache'):
|
||||
self._simplex_portion_cache = self.montecarlo_proportion()
|
||||
return self._simplex_portion_cache
|
||||
|
||||
@lru_cache
|
||||
def montecarlo_proportion(self, n_trials=10_000):
|
||||
"""
|
||||
Estimates, via a Monte Carlo approach, the fraction of the simplex covered by the region. This is carried
|
||||
|
|
@ -84,10 +83,13 @@ class ConfidenceRegionABC(ABC):
|
|||
|
||||
:return: float in [0,1]
|
||||
"""
|
||||
with qp.util.temp_seed(0):
|
||||
uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials)
|
||||
proportion = np.clip(self.coverage(uniform_simplex), 0., 1.)
|
||||
return proportion
|
||||
if not hasattr(self, '_montecarlo_proportion_cache'):
|
||||
self._montecarlo_proportion_cache = {}
|
||||
if n_trials not in self._montecarlo_proportion_cache:
|
||||
with qp.util.temp_seed(0):
|
||||
uniform_simplex = F.uniform_simplex_sampling(n_classes=self.ndim(), size=n_trials)
|
||||
self._montecarlo_proportion_cache[n_trials] = np.clip(self.coverage(uniform_simplex), 0., 1.)
|
||||
return self._montecarlo_proportion_cache[n_trials]
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
|
|
@ -446,7 +448,7 @@ class ConfidenceEllipseSimplex(ConfidenceRegionABC):
|
|||
|
||||
try:
|
||||
self.precision_matrix_ = np.linalg.pinv(self.cov_)
|
||||
except:
|
||||
except np.linalg.LinAlgError:
|
||||
self.precision_matrix_ = None
|
||||
|
||||
self.dim = samples.shape[-1]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import itertools
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from typing import Union, List
|
||||
import numpy as np
|
||||
|
|
@ -26,65 +27,6 @@ else:
|
|||
QuaNet = "QuaNet is not available due to missing torch package"
|
||||
|
||||
|
||||
class MedianEstimator2(BinaryQuantifier):
|
||||
"""
|
||||
This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
|
||||
estimation returned by differently (hyper)parameterized base quantifiers.
|
||||
The median of unit-vectors is only guaranteed to be a unit-vector for n=2 dimensions,
|
||||
i.e., in cases of binary quantification.
|
||||
|
||||
:param base_quantifier: the base, binary quantifier
|
||||
:param random_state: a seed to be set before fitting any base quantifier (default None)
|
||||
:param param_grid: the grid or parameters towards which the median will be computed
|
||||
:param n_jobs: number of parllel workes
|
||||
"""
|
||||
def __init__(self, base_quantifier: BinaryQuantifier, param_grid: dict, random_state=None, n_jobs=None):
|
||||
self.base_quantifier = base_quantifier
|
||||
self.param_grid = param_grid
|
||||
self.random_state = random_state
|
||||
self.n_jobs = qp._get_njobs(n_jobs)
|
||||
|
||||
def get_params(self, deep=True):
|
||||
return self.base_quantifier.get_params(deep)
|
||||
|
||||
def set_params(self, **params):
|
||||
self.base_quantifier.set_params(**params)
|
||||
|
||||
def _delayed_fit(self, args):
|
||||
with qp.util.temp_seed(self.random_state):
|
||||
params, X, y = args
|
||||
model = deepcopy(self.base_quantifier)
|
||||
model.set_params(**params)
|
||||
model.fit(X, y)
|
||||
return model
|
||||
|
||||
def fit(self, X, y):
|
||||
self._check_binary(y, self.__class__.__name__)
|
||||
|
||||
configs = qp.model_selection.expand_grid(self.param_grid)
|
||||
self.models = qp.util.parallel(
|
||||
self._delayed_fit,
|
||||
((params, X, y) for params in configs),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs
|
||||
)
|
||||
return self
|
||||
|
||||
def _delayed_predict(self, args):
|
||||
model, instances = args
|
||||
return model.predict(instances)
|
||||
|
||||
def predict(self, X):
|
||||
prev_preds = qp.util.parallel(
|
||||
self._delayed_predict,
|
||||
((model, X) for model in self.models),
|
||||
seed=qp.environ.get('_R_SEED', None),
|
||||
n_jobs=self.n_jobs
|
||||
)
|
||||
prev_preds = np.asarray(prev_preds)
|
||||
return np.median(prev_preds, axis=0)
|
||||
|
||||
|
||||
class MedianEstimator(BinaryQuantifier):
|
||||
"""
|
||||
This method is a meta-quantifier that returns, as the estimated class prevalence values, the median of the
|
||||
|
|
@ -213,7 +155,7 @@ class Ensemble(BaseQuantifier):
|
|||
|
||||
def _sout(self, msg):
|
||||
if self.verbose:
|
||||
print('[Ensemble]' + msg)
|
||||
logging.getLogger(__name__).info('[Ensemble] ' + msg)
|
||||
|
||||
def fit(self, X, y):
|
||||
|
||||
|
|
@ -402,7 +344,7 @@ def _select_k(elements, order, k):
|
|||
def _delayed_new_instance(args):
|
||||
base_quantifier, data, val_split, prev, posteriors, keep_samples, verbose, sample_size = args
|
||||
if verbose:
|
||||
print(f'\tfit-start for prev {F.strprev(prev)}, sample_size={sample_size}')
|
||||
logging.getLogger(__name__).info(f'fit-start for prev {F.strprev(prev)}, sample_size={sample_size}')
|
||||
model = deepcopy(base_quantifier)
|
||||
|
||||
if val_split is not None:
|
||||
|
|
@ -422,7 +364,7 @@ def _delayed_new_instance(args):
|
|||
tr_distribution = get_probability_distribution(posteriors[sample_index]) if (posteriors is not None) else None
|
||||
|
||||
if verbose:
|
||||
print(f'\t--fit-ended for prev {F.strprev(prev)}')
|
||||
logging.getLogger(__name__).info(f'fit-ended for prev {F.strprev(prev)}')
|
||||
|
||||
return (model, tr_prevalence, tr_distribution, sample if keep_samples else None)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import itertools
|
||||
import logging
|
||||
import signal
|
||||
from copy import deepcopy
|
||||
from enum import Enum
|
||||
|
|
@ -91,7 +92,7 @@ class GridSearchQ(BaseQuantifier):
|
|||
|
||||
def _sout(self, msg):
|
||||
if self.verbose:
|
||||
print(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}')
|
||||
logging.getLogger(__name__).info(f'[{self.__class__.__name__}:{self.model.__class__.__name__}]: {msg}')
|
||||
|
||||
def __check_error_measure(self, error):
|
||||
if error in qp.error.QUANTIFICATION_ERROR:
|
||||
|
|
|
|||
|
|
@ -483,7 +483,6 @@ def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs
|
|||
best_bucket_methods.append(method_order[method_index])
|
||||
best_methods.append(best_bucket_methods)
|
||||
salient_methods.update(best_bucket_methods)
|
||||
print(best_bucket_methods)
|
||||
|
||||
if binning=='isomerous':
|
||||
fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods)))
|
||||
|
|
@ -827,7 +826,6 @@ def calibration_plot(prob_classifier, X, y, nbins=10, savepath=None):
|
|||
pred_y = posteriors>=0.5
|
||||
bins = np.linspace(0, 1, nbins + 1)
|
||||
binned_values = np.digitize(posteriors, bins, right=False)
|
||||
print(np.unique(binned_values))
|
||||
correct = pred_y == y
|
||||
bin_centers = (bins[:-1] + bins[1:]) / 2
|
||||
bins_names = np.arange(nbins)
|
||||
|
|
|
|||
|
|
@ -445,11 +445,6 @@ class DirichletProtocol(OnLabelledCollectionProtocol):
|
|||
|
||||
def __init__(self, data: LabelledCollection, alpha, sample_size=None, repeats=100, random_state=0,
|
||||
return_type='sample_prev'):
|
||||
#assert ((isinstance(alpha, str) and alpha == 'uniform') or
|
||||
# isinstance(alpha, Number) or
|
||||
# (isinstance(alpha, Iterable) and all(isinstance(v, Number) for v in alpha))), \
|
||||
# f'wrong type for {alpha=}; expected "uniform", a real scalar, or an array-like of real values'
|
||||
|
||||
n_classes = data.n_classes
|
||||
if isinstance(alpha, str) and alpha == 'uniform':
|
||||
self.alpha = np.ones(n_classes, dtype=float)
|
||||
|
|
@ -464,7 +459,6 @@ class DirichletProtocol(OnLabelledCollectionProtocol):
|
|||
|
||||
super(DirichletProtocol, self).__init__(random_state)
|
||||
self.data = data
|
||||
#self.alpha = alpha
|
||||
self.sample_size = qp._get_sample_size(sample_size)
|
||||
self.repeats = repeats
|
||||
self.random_state = random_state
|
||||
|
|
|
|||
Loading…
Reference in New Issue