running comparison

main.py

@@ -8,7 +8,7 @@ from src.util.results_csv import CSVlog
 from src.view_generators import *
 
 import os
-os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
 
 
 def main(args):
@@ -25,6 +25,14 @@ def main(args):
     lX, ly = data.training()
     lXte, lyte = data.test()
 
+    # TODO: debug settings
+    print(f'\n[Running on DEBUG mode - samples per language are reduced to 50 max!]\n')
+    lX = {k: v[:50] for k, v in lX.items()}
+    ly = {k: v[:50] for k, v in ly.items()}
+    lXte = {k: v[:50] for k, v in lXte.items()}
+    lyte = {k: v[:50] for k, v in lyte.items()}
+
+
     # Init multilingualIndex - mandatory when deploying Neural View Generators...
     if args.gru_embedder or args.bert_embedder:
         multilingualIndex = MultilingualIndex()
@@ -101,6 +109,8 @@ def main(args):
     time_tr = round(time.time() - time_init, 3)
     print(f'Training completed in {time_tr} seconds!')
 
+    exit('[Exiting DEBUG session without testing overall architecture!]')
+
     # Testing ----------------------------------------
     print('\n[Testing Generalized Funnelling]')
     time_te = time.time()

src/models/pl_bert.py

@@ -2,8 +2,6 @@ import pytorch_lightning as pl
 import torch
 from torch.optim.lr_scheduler import StepLR
 from transformers import BertForSequenceClassification, AdamW
-import numpy as np
-import csv
 
 from src.util.common import define_pad_length, pad
 from src.util.pl_metrics import CustomF1, CustomK
@@ -11,7 +9,7 @@ from src.util.pl_metrics import CustomF1, CustomK
 
 class BertModel(pl.LightningModule):
 
-    def __init__(self, output_size, stored_path, gpus=None, manual_log=False):
+    def __init__(self, output_size, stored_path, gpus=None):
         """
         Init Bert model.
         :param output_size:
@@ -41,17 +39,6 @@ class BertModel(pl.LightningModule):
                                                                       output_hidden_states=True)
         self.save_hyperparameters()
 
-        # Manual logging settings
-        self.manual_log = manual_log
-        if self.manual_log:
-            from src.util.file import create_if_not_exist
-            self.csv_file = f'csv_logs/bert/bert_manual_log_v{self._version}.csv'
-            with open(self.csv_file, 'x') as handler:
-                writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
-                writer.writerow(['tr_loss', 'va_loss', 'va_macroF1', 'va_microF1', 'va_macroK', 'va_microK'])
-        self.csv_metrics = {'tr_loss': [], 'va_loss': [], 'va_macroF1': [],
-                            'va_microF1': [], 'va_macroK': [], 'va_microK': []}
-
     def forward(self, X):
         logits = self.bert(X)
         return logits
@@ -72,10 +59,14 @@ class BertModel(pl.LightningModule):
         self.log('train-microF1', microF1,  on_step=False, on_epoch=True, prog_bar=False, logger=True)
         self.log('train-macroK', macroK,    on_step=False, on_epoch=True, prog_bar=False, logger=True)
         self.log('train-microK', microK,    on_step=False, on_epoch=True, prog_bar=False, logger=True)
-        lX, ly = self._reconstruct_dict(predictions, y, batch_langs)
+        return {'loss': loss}
-        return {'loss': loss, 'pred': lX, 'target': ly}
+        # lX, ly = self._reconstruct_dict(predictions, y, batch_langs)
+        # return {'loss': loss, 'pred': lX, 'target': ly}
 
+    """
     def training_epoch_end(self, outputs):
+        pass
+        
         langs = []
         for output in outputs:
             langs.extend(list(output['pred'].keys()))
@@ -114,6 +105,7 @@ class BertModel(pl.LightningModule):
             tr_epoch_loss = np.average([out['loss'].item() for out in outputs])
             self.csv_metrics['tr_loss'].append(tr_epoch_loss)
             self.save_manual_logs()
+    """
 
     def validation_step(self, val_batch, batch_idx):
         X, y, batch_langs = val_batch
@@ -130,16 +122,23 @@ class BertModel(pl.LightningModule):
         self.log('val-microF1', microF1,    on_step=False, on_epoch=True, prog_bar=True, logger=True)
         self.log('val-macroK', macroK,      on_step=False, on_epoch=True, prog_bar=True, logger=True)
         self.log('val-microK', microK,      on_step=False, on_epoch=True, prog_bar=True, logger=True)
-
-        if self.manual_log:
-            # Manual logging to csv
-            self.csv_metrics['va_loss'].append(loss.item())
-            self.csv_metrics['va_macroF1'].append(macroF1.item())
-            self.csv_metrics['va_microF1'].append(microF1.item())
-            self.csv_metrics['va_macroK'].append(macroK.item())
-            self.csv_metrics['va_microK'].append(microK.item())
-
         return {'loss': loss}
+        # return {'loss': loss, 'pred': predictions, 'target': y}
+
+    # def validation_epoch_end(self, outputs):
+    #     all_pred = []
+    #     all_tar = []
+    #     for output in outputs:
+    #         all_pred.append(output['pred'].cpu().numpy())
+    #         all_tar.append(output['target'].cpu().numpy())
+    #     all_pred = np.vstack(all_pred)
+    #     all_tar = np.vstack(all_tar)
+    #     all_pred = {'all': all_pred}
+    #     all_tar = {'all': all_tar}
+    #     res = evaluate(all_tar, all_pred)
+    #     res = [elem for elem in res.values()]
+    #     res = np.average(res, axis=0)
+    #     print(f'\n{res}')
 
     def test_step(self, test_batch, batch_idx):
         X, y, batch_langs = test_batch
@@ -209,14 +208,3 @@ class BertModel(pl.LightningModule):
         for k, v in reconstructed_y.items():
             reconstructed_y[k] = torch.cat(v).view(-1, predictions.shape[1])
         return reconstructed_x, reconstructed_y
-
-    def save_manual_logs(self):
-        if self.global_step == 0:
-            return
-        with open(self.csv_file, 'a', newline='\n') as handler:
-            writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
-            writer.writerow([np.average(self.csv_metrics['tr_loss']), np.average(self.csv_metrics['va_loss']),
-                            np.average(self.csv_metrics['va_macroF1']), np.average(self.csv_metrics['va_microF1']),
-                            np.average(self.csv_metrics['va_macroK']), np.average(self.csv_metrics['va_microK'])])
-
-

src/util/metrics.py

@@ -118,6 +118,7 @@ def hard_single_metric_statistics(true_labels, predicted_labels):
 
 def macro_average(true_labels, predicted_labels, metric, metric_statistics=hard_single_metric_statistics):
     true_labels, predicted_labels, nC = __check_consistency_and_adapt(true_labels, predicted_labels)
+    _tmp = [metric(metric_statistics(true_labels[:, c], predicted_labels[:, c])) for c in range(nC)]
     return np.mean([metric(metric_statistics(true_labels[:, c], predicted_labels[:, c])) for c in range(nC)])
 
 

src/util/pl_metrics.py

@@ -68,7 +68,7 @@ class CustomF1(Metric):
                 if den > 0:
                     class_specific.append(num / den)
                 else:
-                    class_specific.append(1.)
+                    class_specific.append(torch.FloatTensor([1.]))
             average = torch.sum(torch.Tensor(class_specific))/self.num_classes
             return average.to(self.device)
 

src/view_generators.py

@@ -15,12 +15,12 @@ This module contains the view generators that take care of computing the view sp
 
 - View generator (-b): generates document embedding via mBERT model.
 """
+import torch
 from abc import ABC, abstractmethod
-import src.util.disable_sklearn_warnings
 # from time import time
 
 from pytorch_lightning import Trainer
-from pytorch_lightning.loggers import TensorBoardLogger
+from pytorch_lightning.loggers import TensorBoardLogger, CSVLogger
 from pytorch_lightning.callbacks.early_stopping import EarlyStopping
 from pytorch_lightning.callbacks.lr_monitor import LearningRateMonitor
 
@@ -448,9 +448,13 @@ class BertGen(ViewGen):
         self.stored_path = stored_path
         self.model = self._init_model()
         self.patience = patience
-        self.logger = TensorBoardLogger(save_dir='tb_logs', name='bert', default_hp_metric=False)
+        # self.logger = TensorBoardLogger(save_dir='tb_logs', name='bert', default_hp_metric=False)
+        self.logger = CSVLogger(save_dir='csv_logs', name='bert')
         self.early_stop_callback = EarlyStopping(monitor='val-macroF1', min_delta=0.00,
-                                                 patience=self.patience, verbose=True, mode='max')
+                                                 patience=self.patience, verbose=False, mode='max')
+
+        # modifying EarlyStopping global var in order to compute >= with respect to the best score
+        self.early_stop_callback.mode_dict['max'] = torch.ge
 
         # Zero shot parameters
         self.zero_shot = zero_shot
@@ -476,7 +480,7 @@ class BertGen(ViewGen):
         self.multilingualIndex.train_val_split(val_prop=0.2, max_val=2000, seed=1)
         bertDataModule = BertDataModule(self.multilingualIndex, batchsize=self.batch_size, max_len=512,
                                         zero_shot=self.zero_shot, zscl_langs=self.train_langs,
-                                        debug=True, max_samples=50)     # todo: debug=True -> DEBUG setting
+                                        debug=False, max_samples=50)
 
         if self.zero_shot:
             print(f'# Zero-shot setting! Training langs will be set to: {sorted(self.train_langs)}')