diff --git a/main.py b/main.py index 6095d43..e59c57b 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ from src.util.results_csv import CSVlog from src.view_generators import * import os -os.environ["CUDA_VISIBLE_DEVICES"] = "1" +os.environ["CUDA_VISIBLE_DEVICES"] = "0" def main(args): @@ -25,6 +25,14 @@ def main(args): lX, ly = data.training() lXte, lyte = data.test() + # TODO: debug settings + print(f'\n[Running on DEBUG mode - samples per language are reduced to 50 max!]\n') + lX = {k: v[:50] for k, v in lX.items()} + ly = {k: v[:50] for k, v in ly.items()} + lXte = {k: v[:50] for k, v in lXte.items()} + lyte = {k: v[:50] for k, v in lyte.items()} + + # Init multilingualIndex - mandatory when deploying Neural View Generators... if args.gru_embedder or args.bert_embedder: multilingualIndex = MultilingualIndex() @@ -101,6 +109,8 @@ def main(args): time_tr = round(time.time() - time_init, 3) print(f'Training completed in {time_tr} seconds!') + exit('[Exiting DEBUG session without testing overall architecture!]') + # Testing ---------------------------------------- print('\n[Testing Generalized Funnelling]') time_te = time.time() diff --git a/src/models/pl_bert.py b/src/models/pl_bert.py index e2aeb56..37b3df4 100644 --- a/src/models/pl_bert.py +++ b/src/models/pl_bert.py @@ -2,8 +2,6 @@ import pytorch_lightning as pl import torch from torch.optim.lr_scheduler import StepLR from transformers import BertForSequenceClassification, AdamW -import numpy as np -import csv from src.util.common import define_pad_length, pad from src.util.pl_metrics import CustomF1, CustomK @@ -11,7 +9,7 @@ from src.util.pl_metrics import CustomF1, CustomK class BertModel(pl.LightningModule): - def __init__(self, output_size, stored_path, gpus=None, manual_log=False): + def __init__(self, output_size, stored_path, gpus=None): """ Init Bert model. :param output_size: @@ -41,17 +39,6 @@ class BertModel(pl.LightningModule): output_hidden_states=True) self.save_hyperparameters() - # Manual logging settings - self.manual_log = manual_log - if self.manual_log: - from src.util.file import create_if_not_exist - self.csv_file = f'csv_logs/bert/bert_manual_log_v{self._version}.csv' - with open(self.csv_file, 'x') as handler: - writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) - writer.writerow(['tr_loss', 'va_loss', 'va_macroF1', 'va_microF1', 'va_macroK', 'va_microK']) - self.csv_metrics = {'tr_loss': [], 'va_loss': [], 'va_macroF1': [], - 'va_microF1': [], 'va_macroK': [], 'va_microK': []} - def forward(self, X): logits = self.bert(X) return logits @@ -72,10 +59,14 @@ class BertModel(pl.LightningModule): self.log('train-microF1', microF1, on_step=False, on_epoch=True, prog_bar=False, logger=True) self.log('train-macroK', macroK, on_step=False, on_epoch=True, prog_bar=False, logger=True) self.log('train-microK', microK, on_step=False, on_epoch=True, prog_bar=False, logger=True) - lX, ly = self._reconstruct_dict(predictions, y, batch_langs) - return {'loss': loss, 'pred': lX, 'target': ly} + return {'loss': loss} + # lX, ly = self._reconstruct_dict(predictions, y, batch_langs) + # return {'loss': loss, 'pred': lX, 'target': ly} + """ def training_epoch_end(self, outputs): + pass + langs = [] for output in outputs: langs.extend(list(output['pred'].keys())) @@ -114,6 +105,7 @@ class BertModel(pl.LightningModule): tr_epoch_loss = np.average([out['loss'].item() for out in outputs]) self.csv_metrics['tr_loss'].append(tr_epoch_loss) self.save_manual_logs() + """ def validation_step(self, val_batch, batch_idx): X, y, batch_langs = val_batch @@ -130,16 +122,23 @@ class BertModel(pl.LightningModule): self.log('val-microF1', microF1, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-macroK', macroK, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-microK', microK, on_step=False, on_epoch=True, prog_bar=True, logger=True) - - if self.manual_log: - # Manual logging to csv - self.csv_metrics['va_loss'].append(loss.item()) - self.csv_metrics['va_macroF1'].append(macroF1.item()) - self.csv_metrics['va_microF1'].append(microF1.item()) - self.csv_metrics['va_macroK'].append(macroK.item()) - self.csv_metrics['va_microK'].append(microK.item()) - return {'loss': loss} + # return {'loss': loss, 'pred': predictions, 'target': y} + + # def validation_epoch_end(self, outputs): + # all_pred = [] + # all_tar = [] + # for output in outputs: + # all_pred.append(output['pred'].cpu().numpy()) + # all_tar.append(output['target'].cpu().numpy()) + # all_pred = np.vstack(all_pred) + # all_tar = np.vstack(all_tar) + # all_pred = {'all': all_pred} + # all_tar = {'all': all_tar} + # res = evaluate(all_tar, all_pred) + # res = [elem for elem in res.values()] + # res = np.average(res, axis=0) + # print(f'\n{res}') def test_step(self, test_batch, batch_idx): X, y, batch_langs = test_batch @@ -209,14 +208,3 @@ class BertModel(pl.LightningModule): for k, v in reconstructed_y.items(): reconstructed_y[k] = torch.cat(v).view(-1, predictions.shape[1]) return reconstructed_x, reconstructed_y - - def save_manual_logs(self): - if self.global_step == 0: - return - with open(self.csv_file, 'a', newline='\n') as handler: - writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) - writer.writerow([np.average(self.csv_metrics['tr_loss']), np.average(self.csv_metrics['va_loss']), - np.average(self.csv_metrics['va_macroF1']), np.average(self.csv_metrics['va_microF1']), - np.average(self.csv_metrics['va_macroK']), np.average(self.csv_metrics['va_microK'])]) - - diff --git a/src/util/metrics.py b/src/util/metrics.py index 7a6079e..d867671 100644 --- a/src/util/metrics.py +++ b/src/util/metrics.py @@ -118,6 +118,7 @@ def hard_single_metric_statistics(true_labels, predicted_labels): def macro_average(true_labels, predicted_labels, metric, metric_statistics=hard_single_metric_statistics): true_labels, predicted_labels, nC = __check_consistency_and_adapt(true_labels, predicted_labels) + _tmp = [metric(metric_statistics(true_labels[:, c], predicted_labels[:, c])) for c in range(nC)] return np.mean([metric(metric_statistics(true_labels[:, c], predicted_labels[:, c])) for c in range(nC)]) diff --git a/src/util/pl_metrics.py b/src/util/pl_metrics.py index 765a6a2..f2e5557 100644 --- a/src/util/pl_metrics.py +++ b/src/util/pl_metrics.py @@ -68,7 +68,7 @@ class CustomF1(Metric): if den > 0: class_specific.append(num / den) else: - class_specific.append(1.) + class_specific.append(torch.FloatTensor([1.])) average = torch.sum(torch.Tensor(class_specific))/self.num_classes return average.to(self.device) diff --git a/src/view_generators.py b/src/view_generators.py index 858f751..9103f2b 100644 --- a/src/view_generators.py +++ b/src/view_generators.py @@ -15,12 +15,12 @@ This module contains the view generators that take care of computing the view sp - View generator (-b): generates document embedding via mBERT model. """ +import torch from abc import ABC, abstractmethod -import src.util.disable_sklearn_warnings # from time import time from pytorch_lightning import Trainer -from pytorch_lightning.loggers import TensorBoardLogger +from pytorch_lightning.loggers import TensorBoardLogger, CSVLogger from pytorch_lightning.callbacks.early_stopping import EarlyStopping from pytorch_lightning.callbacks.lr_monitor import LearningRateMonitor @@ -448,9 +448,13 @@ class BertGen(ViewGen): self.stored_path = stored_path self.model = self._init_model() self.patience = patience - self.logger = TensorBoardLogger(save_dir='tb_logs', name='bert', default_hp_metric=False) + # self.logger = TensorBoardLogger(save_dir='tb_logs', name='bert', default_hp_metric=False) + self.logger = CSVLogger(save_dir='csv_logs', name='bert') self.early_stop_callback = EarlyStopping(monitor='val-macroF1', min_delta=0.00, - patience=self.patience, verbose=True, mode='max') + patience=self.patience, verbose=False, mode='max') + + # modifying EarlyStopping global var in order to compute >= with respect to the best score + self.early_stop_callback.mode_dict['max'] = torch.ge # Zero shot parameters self.zero_shot = zero_shot @@ -476,7 +480,7 @@ class BertGen(ViewGen): self.multilingualIndex.train_val_split(val_prop=0.2, max_val=2000, seed=1) bertDataModule = BertDataModule(self.multilingualIndex, batchsize=self.batch_size, max_len=512, zero_shot=self.zero_shot, zscl_langs=self.train_langs, - debug=True, max_samples=50) # todo: debug=True -> DEBUG setting + debug=False, max_samples=50) if self.zero_shot: print(f'# Zero-shot setting! Training langs will be set to: {sorted(self.train_langs)}')