1
0
Fork 0

adapting everything to the new format

This commit is contained in:
Alejandro Moreo Fernandez 2021-11-08 18:01:49 +01:00
parent f63575ff55
commit 238a30520c
7 changed files with 81 additions and 72 deletions

View File

@ -40,14 +40,17 @@ print(f'training matrix shape: {train.instances.shape}')
true_prevalence = ResultSubmission.load(T1A_devprevalence_path)
param_grid = {'C': np.logspace(-3,3,7), 'class_weight': ['balanced', None]}
param_grid = {
'C': np.logspace(-3,3,7),
'class_weight': ['balanced', None]
}
def gen_samples():
return gen_load_samples_T1(T1A_devvectors_path, nF, ground_truth_path=T1A_devprevalence_path, return_id=False)
for quantifier in [CC, ACC, PCC, PACC, EMQ, HDy]:
for quantifier in [CC]: #, ACC, PCC, PACC, EMQ, HDy]:
#classifier = CalibratedClassifierCV(LogisticRegression(), n_jobs=-1)
classifier = LogisticRegression()
model = quantifier(classifier)
@ -66,7 +69,7 @@ for quantifier in [CC, ACC, PCC, PACC, EMQ, HDy]:
print(f'{quantifier_name} mae={model.best_score_:.3f} (params: {model.best_params_})')
pickle.dump(model.best_model(),
open(os.path.join(models_path, quantifier_name+'.modsel.pkl'), 'wb'),
open(os.path.join(models_path, quantifier_name+'.pkl'), 'wb'),
protocol=pickle.HIGHEST_PROTOCOL)

View File

@ -1,7 +1,12 @@
DEV_SAMPLES = 1000
TEST_SAMPLES = 5000
TXA_SAMPLE_SIZE = 250
TXB_SAMPLE_SIZE = 250
T1A_SAMPLE_SIZE = 250
T1B_SAMPLE_SIZE = 1000
T2A_SAMPLE_SIZE = 250
T2B_SAMPLE_SIZE = 1000
ERROR_TOL = 1E-3

View File

@ -80,21 +80,30 @@ def gen_load_samples_T2B(path_dir:str, ground_truth_path:str = None):
class ResultSubmission:
def __init__(self, categories: List[str]):
if not isinstance(categories, list) or len(categories) < 2:
raise TypeError('wrong format for categories; a list with at least two category names (str) was expected')
self.categories = categories
self.df = pd.DataFrame(columns=list(categories))
self.df.index.rename('id', inplace=True)
def __init__(self):
self.df = None
def __init_df(self, categories:int):
if not isinstance(categories, int) or categories < 2:
raise TypeError('wrong format for categories: an int (>=2) was expected')
df = pd.DataFrame(columns=list(range(categories)))
df.index.set_names('id', inplace=True)
self.df = df
@property
def n_categories(self):
return len(self.df.columns.values)
def add(self, sample_id:int, prevalence_values:np.ndarray):
if not isinstance(sample_id, int):
raise TypeError(f'error: expected int for sample_sample, found {type(sample_id)}')
if not isinstance(prevalence_values, np.ndarray):
raise TypeError(f'error: expected np.ndarray for prevalence_values, found {type(prevalence_values)}')
if self.df is None:
self.__init_df(categories=len(prevalence_values))
if sample_id in self.df.index.values:
raise ValueError(f'error: prevalence values for "{sample_id}" already added')
if prevalence_values.ndim!=1 and prevalence_values.size != len(self.categories):
if prevalence_values.ndim!=1 and prevalence_values.size != self.n_categories:
raise ValueError(f'error: wrong shape found for prevalence vector {prevalence_values}')
if (prevalence_values<0).any() or (prevalence_values>1).any():
raise ValueError(f'error: prevalence values out of range [0,1] for "{sample_id}"')
@ -102,9 +111,7 @@ class ResultSubmission:
raise ValueError(f'error: prevalence values do not sum up to one for "{sample_id}"'
f'(error tolerance {constants.ERROR_TOL})')
# new_entry = dict([('id', sample_id)] + [(col_i, prev_i) for col_i, prev_i in enumerate(prevalence_values)])
new_entry = pd.DataFrame(prevalence_values.reshape(1,2), index=[sample_id], columns=self.df.columns)
self.df = self.df.append(new_entry, ignore_index=False)
self.df.loc[sample_id] = prevalence_values
def __len__(self):
return len(self.df)
@ -112,7 +119,7 @@ class ResultSubmission:
@classmethod
def load(cls, path: str) -> 'ResultSubmission':
df = ResultSubmission.check_file_format(path)
r = ResultSubmission(categories=df.columns.values.tolist())
r = ResultSubmission()
r.df = df
return r
@ -120,16 +127,15 @@ class ResultSubmission:
ResultSubmission.check_dataframe_format(self.df)
self.df.to_csv(path)
def prevalence(self, sample_name:str):
sel = self.df.loc[self.df['filename'] == sample_name]
def prevalence(self, sample_id:int):
sel = self.df.loc[sample_id]
if sel.empty:
return None
else:
return sel.loc[:,self.df.columns[1]:].values.flatten()
return sel.values.flatten()
def iterrows(self):
for index, row in self.df.iterrows():
# filename = row.filename
prevalence = row.values.flatten()
yield index, prevalence
@ -146,10 +152,11 @@ class ResultSubmission:
if df.index.name != 'id' or len(df.columns) < 2:
raise ValueError(f'wrong header{hint_path}, '
f'the format of the header should be "id,<cat_1>,...,<cat_n>"')
f'the format of the header should be "id,0,...,n-1", '
f'where n is the number of categories')
if [int(ci) for ci in df.columns.values] != list(range(len(df.columns))):
raise ValueError(f'wrong header{hint_path}, category ids should be 0,1,2,...,n')
raise ValueError(f'wrong header{hint_path}, category ids should be 0,1,2,...,n-1, '
f'where n is the number of categories')
if df.empty:
raise ValueError(f'error{hint_path}: results file is empty')
elif len(df) != constants.DEV_SAMPLES and len(df) != constants.TEST_SAMPLES:
@ -167,9 +174,9 @@ class ResultSubmission:
if unexpected:
raise ValueError(f'there are {len(missing)} unexpected ids{hint_path}: {sorted(unexpected)}')
for category_name in df.columns:
if (df[category_name] < 0).any() or (df[category_name] > 1).any():
raise ValueError(f'error{hint_path} column "{category_name}" contains values out of range [0,1]')
for category_id in df.columns:
if (df[category_id] < 0).any() or (df[category_id] > 1).any():
raise ValueError(f'error{hint_path} column "{category_id}" contains values out of range [0,1]')
prevs = df.values
round_errors = np.abs(prevs.sum(axis=-1) - 1.) > constants.ERROR_TOL
@ -180,13 +187,6 @@ class ResultSubmission:
return df
def sort_categories(self):
self.df = self.df.reindex([self.df.columns[0]] + sorted(self.df.columns[1:]), axis=1)
self.categories = sorted(self.categories)
def filenames(self):
return self.df.filename.values
def evaluate_submission(true_prevs: ResultSubmission, predicted_prevs: ResultSubmission, sample_size=None, average=True):
if sample_size is None:
@ -199,18 +199,19 @@ def evaluate_submission(true_prevs: ResultSubmission, predicted_prevs: ResultSub
if len(true_prevs) != len(predicted_prevs):
raise ValueError(f'size mismatch, ground truth file has {len(true_prevs)} entries '
f'while the file of predictions contain {len(predicted_prevs)} entries')
true_prevs.sort_categories()
predicted_prevs.sort_categories()
if true_prevs.categories != predicted_prevs.categories:
if true_prevs.n_categories != predicted_prevs.n_categories:
raise ValueError(f'these result files are not comparable since the categories are different: '
f'true={true_prevs.categories} vs. predictions={predicted_prevs.categories}')
f'true={true_prevs.n_categories} categories vs. '
f'predictions={predicted_prevs.n_categories} categories')
ae, rae = [], []
for sample_name, true_prevalence in true_prevs.iterrows():
pred_prevalence = predicted_prevs.prevalence(sample_name)
for sample_id, true_prevalence in true_prevs.iterrows():
pred_prevalence = predicted_prevs.prevalence(sample_id)
ae.append(qp.error.ae(true_prevalence, pred_prevalence))
rae.append(qp.error.rae(true_prevalence, pred_prevalence, eps=1./(2*sample_size)))
ae = np.asarray(ae)
rae = np.asarray(rae)
if average:
return ae.mean(), rae.mean()
else:
@ -224,3 +225,4 @@ def evaluate_submission(true_prevs: ResultSubmission, predicted_prevs: ResultSub

View File

@ -9,8 +9,10 @@ LeQua2022 Official evaluation script
"""
def main(args):
if args.task in {'T1A'}:
qp.environ['SAMPLE_SIZE'] = constants.T1A_SAMPLE_SIZE
if args.task in {'T1A', 'T2A'}:
qp.environ['SAMPLE_SIZE'] = constants.TXA_SAMPLE_SIZE
if args.task in {'T1B', 'T2B'}:
qp.environ['SAMPLE_SIZE'] = constants.TXB_SAMPLE_SIZE
true_prev = ResultSubmission.load(args.true_prevalences)
pred_prev = ResultSubmission.load(args.pred_prevalences)
mae, mrae = evaluate_submission(true_prev, pred_prev)

View File

@ -22,13 +22,13 @@ def main(args):
f'dev samples ({constants.DEV_SAMPLES}) nor with the expected number of '
f'test samples ({constants.TEST_SAMPLES}).')
_, categories = load_category_map(args.catmap)
# _, categories = load_category_map(args.catmap)
# load pickled model
model = pickle.load(open(args.model, 'rb'))
# predictions
predictions = ResultSubmission(categories=list(range(len(categories))))
predictions = ResultSubmission()
for sampleid, sample in tqdm(gen_load_samples_T1(args.samples, args.nf),
desc='predicting', total=nsamples):
predictions.add(sampleid, model.quantify(sample))
@ -48,8 +48,6 @@ if __name__=='__main__':
help='Path to the directory containing the samples')
parser.add_argument('output', metavar='PREDICTIONS-PATH', type=str,
help='Path where to store the predictions file')
parser.add_argument('catmap', metavar='CATEGORY-MAP-PATH', type=str,
help='Path to the category map file')
parser.add_argument('nf', metavar='NUM-FEATURES', type=int,
help='Number of features seen during training')
args = parser.parse_args()

View File

@ -81,7 +81,7 @@ def natural_prevalence_prediction(
def gen_prevalence_prediction(model: BaseQuantifier, gen_fn: Callable, eval_budget=None):
if not inspect.isgenerator(gen_fn()):
raise ValueError('param "gen_fun" is not a generator')
raise ValueError('param "gen_fun" is not a callable returning a generator')
if not isinstance(eval_budget, int):
eval_budget = -1

View File

@ -15,7 +15,7 @@ class GridSearchQ(BaseQuantifier):
def __init__(self,
model: BaseQuantifier,
param_grid: dict,
sample_size: int,
sample_size: Union[int, None],
protocol='app',
n_prevpoints: int = None,
n_repetitions: int = 1,
@ -32,30 +32,33 @@ class GridSearchQ(BaseQuantifier):
protocol for quantification.
:param model: the quantifier to optimize
:param param_grid: a dictionary with keys the parameter names and values the list of values to explore for
:param sample_size: the size of the samples to extract from the validation set
that particular parameter
:param protocol: either 'app' for the artificial prevalence protocol, or 'npp' for the natural prevalence
protocol
:param n_prevpoints: if specified, indicates the number of equally distant point to extract from the interval
:param sample_size: the size of the samples to extract from the validation set (ignored if protocl='gen')
:param protocol: either 'app' for the artificial prevalence protocol, 'npp' for the natural prevalence
protocol, or 'gen' for using a custom sampling generator function
:param n_prevpoints: if specified, indicates the number of equally distant points to extract from the interval
[0,1] in order to define the prevalences of the samples; e.g., if n_prevpoints=5, then the prevalences for
each class will be explored in [0.00, 0.25, 0.50, 0.75, 1.00]. If not specified, then eval_budget is requested.
Ignored if protocol='npp'.
Ignored if protocol!='app'.
:param n_repetitions: the number of repetitions for each combination of prevalences. This parameter is ignored
if eval_budget is set and is lower than the number of combinations that would be generated using the value
assigned to n_prevpoints (for the current number of classes and n_repetitions)
for the protocol='app' if eval_budget is set and is lower than the number of combinations that would be
generated using the value assigned to n_prevpoints (for the current number of classes and n_repetitions).
Ignored for protocol='npp' and protocol='gen' (use eval_budget for setting a maximum number of samples in
those cases).
:param eval_budget: if specified, sets a ceil on the number of evaluations to perform for each hyper-parameter
combination. For example, if there are 3 classes, n_repetitions=1 and eval_budget=20, then n_prevpoints will be
set to 5, since this will generate 15 different prevalences:
[0, 0, 1], [0, 0.25, 0.75], [0, 0.5, 0.5] ... [1, 0, 0]
Ignored if protocol='npp'.
combination. For example, if protocol='app', there are 3 classes, n_repetitions=1 and eval_budget=20, then
n_prevpoints will be set to 5, since this will generate 15 different prevalences, i.e., [0, 0, 1],
[0, 0.25, 0.75], [0, 0.5, 0.5] ... [1, 0, 0], and since setting it to 6 would generate more than
20. When protocol='gen', indicates the maximum number of samples to generate, but less samples will be
generated if the generator yields less samples.
:param error: an error function (callable) or a string indicating the name of an error function (valid ones
are those in qp.error.QUANTIFICATION_ERROR
:param refit: whether or not to refit the model on the whole labelled collection (training+validation) with
the best chosen hyperparameter combination
the best chosen hyperparameter combination. Ignored if protocol='gen'
:param val_split: either a LabelledCollection on which to test the performance of the different settings, or
a float in [0,1] indicating the proportion of labelled data to extract from the training set
a float in [0,1] indicating the proportion of labelled data to extract from the training set, or a callable
returning a generator function each time it is invoked (only for protocol='gen').
:param n_jobs: number of parallel jobs
:param random_seed: set the seed of the random generator to replicate experiments
:param random_seed: set the seed of the random generator to replicate experiments. Ignored if protocol='gen'.
:param timeout: establishes a timer (in seconds) for each of the hyperparameters configurations being tested.
Whenever a run takes longer than this timer, that configuration will be ignored. If all configurations end up
being ignored, a TimeoutError exception is raised. If -1 (default) then no time bound is set.
@ -79,17 +82,13 @@ class GridSearchQ(BaseQuantifier):
'unknown protocol: valid ones are "app" or "npp" for the "artificial" or the "natural" prevalence ' \
'protocols. Use protocol="gen" when passing a generator function thorough val_split that yields a ' \
'sample (instances) and their prevalence (ndarray) at each iteration.'
if self.protocol == 'npp':
if self.n_repetitions is None or self.n_repetitions == 1:
if self.eval_budget is not None:
print(f'[warning] when protocol=="npp" the parameter n_repetitions should be indicated '
f'(and not eval_budget). Setting n_repetitions={self.eval_budget}...')
self.n_repetitions = self.eval_budget
else:
raise ValueError(f'when protocol=="npp" the parameter n_repetitions should be indicated '
f'(and should be >1).')
if self.n_prevpoints is not None:
print('[warning] n_prevpoints has been set along with the npp protocol, and will be ignored')
assert self.eval_budget is None or isinstance(self.eval_budget, int)
if self.protocol in ['npp', 'gen']:
if self.protocol=='npp' and (self.eval_budget is None or self.eval_budget <= 0):
raise ValueError(f'when protocol="npp" the parameter eval_budget should be '
f'indicated (and should be >0).')
if self.n_prevpoints != 1:
print('[warning] n_prevpoints has been set and will be ignored for the selected protocol')
def sout(self, msg):
if self.verbose:
@ -145,7 +144,7 @@ class GridSearchQ(BaseQuantifier):
else:
raise ValueError('unknown protocol')
def fit(self, training: LabelledCollection, val_split: Union[LabelledCollection, float] = None):
def fit(self, training: LabelledCollection, val_split: Union[LabelledCollection, float, Callable] = None):
"""
:param training: the training set on which to optimize the hyperparameters
:param val_split: either a LabelledCollection on which to test the performance of the different settings, or