diff --git a/src/data/datamodule.py b/src/data/datamodule.py index 268bf29..d2191d6 100644 --- a/src/data/datamodule.py +++ b/src/data/datamodule.py @@ -181,7 +181,8 @@ class BertDataModule(RecurrentDataModule): Pytorch Lightning Datamodule to be deployed with BertGen. https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html """ - def __init__(self, multilingualIndex, batchsize=64, max_len=512, zero_shot=False, zscl_langs=None, debug=False): + def __init__(self, multilingualIndex, batchsize=64, max_len=512, zero_shot=False, zscl_langs=None, debug=False, + max_samples=50): """ Init BertDataModule. :param multilingualIndex: MultilingualIndex, it is a dictionary of training and test documents @@ -197,8 +198,9 @@ class BertDataModule(RecurrentDataModule): self.zero_shot = zero_shot self.train_langs = zscl_langs self.debug = debug + self.max_samples = max_samples if self.debug: - print('\n[Running on DEBUG mode - samples per language are reduced to 50 max!]\n') + print(f'\n[Running on DEBUG mode - samples per language are reduced to {self.max_samples} max!]\n') def setup(self, stage=None): if stage == 'fit' or stage is None: @@ -208,8 +210,8 @@ class BertDataModule(RecurrentDataModule): l_train_raw, l_train_target = self.multilingualIndex.l_train_raw() if self.debug: # Debug settings: reducing number of samples - l_train_raw = {l: train[:50] for l, train in l_train_raw.items()} - l_train_target = {l: target[:50] for l, target in l_train_target.items()} + l_train_raw = {l: train[:self.max_samples] for l, train in l_train_raw.items()} + l_train_target = {l: target[:self.max_samples] for l, target in l_train_target.items()} l_train_index = tokenize(l_train_raw, max_len=self.max_len) self.training_dataset = RecurrentDataset(l_train_index, l_train_target, @@ -221,8 +223,8 @@ class BertDataModule(RecurrentDataModule): l_val_raw, l_val_target = self.multilingualIndex.l_val_raw() if self.debug: # Debug settings: reducing number of samples - l_val_raw = {l: train[:50] for l, train in l_val_raw.items()} - l_val_target = {l: target[:50] for l, target in l_val_target.items()} + l_val_raw = {l: train[:self.max_samples] for l, train in l_val_raw.items()} + l_val_target = {l: target[:self.max_samples] for l, target in l_val_target.items()} l_val_index = tokenize(l_val_raw, max_len=self.max_len) self.val_dataset = RecurrentDataset(l_val_index, l_val_target, @@ -235,8 +237,8 @@ class BertDataModule(RecurrentDataModule): l_test_raw, l_test_target = self.multilingualIndex.l_test_raw() if self.debug: # Debug settings: reducing number of samples - l_test_raw = {l: train[:50] for l, train in l_test_raw.items()} - l_test_target = {l: target[:50] for l, target in l_test_target.items()} + l_test_raw = {l: train[:self.max_samples] for l, train in l_test_raw.items()} + l_test_target = {l: target[:self.max_samples] for l, target in l_test_target.items()} l_test_index = tokenize(l_test_raw, max_len=self.max_len) self.test_dataset = RecurrentDataset(l_test_index, l_test_target, diff --git a/src/models/pl_bert.py b/src/models/pl_bert.py index 0a38e9f..e2aeb56 100644 --- a/src/models/pl_bert.py +++ b/src/models/pl_bert.py @@ -2,6 +2,8 @@ import pytorch_lightning as pl import torch from torch.optim.lr_scheduler import StepLR from transformers import BertForSequenceClassification, AdamW +import numpy as np +import csv from src.util.common import define_pad_length, pad from src.util.pl_metrics import CustomF1, CustomK @@ -9,7 +11,7 @@ from src.util.pl_metrics import CustomF1, CustomK class BertModel(pl.LightningModule): - def __init__(self, output_size, stored_path, gpus=None): + def __init__(self, output_size, stored_path, gpus=None, manual_log=False): """ Init Bert model. :param output_size: @@ -39,6 +41,17 @@ class BertModel(pl.LightningModule): output_hidden_states=True) self.save_hyperparameters() + # Manual logging settings + self.manual_log = manual_log + if self.manual_log: + from src.util.file import create_if_not_exist + self.csv_file = f'csv_logs/bert/bert_manual_log_v{self._version}.csv' + with open(self.csv_file, 'x') as handler: + writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) + writer.writerow(['tr_loss', 'va_loss', 'va_macroF1', 'va_microF1', 'va_macroK', 'va_microK']) + self.csv_metrics = {'tr_loss': [], 'va_loss': [], 'va_macroF1': [], + 'va_microF1': [], 'va_macroK': [], 'va_microK': []} + def forward(self, X): logits = self.bert(X) return logits @@ -54,11 +67,11 @@ class BertModel(pl.LightningModule): macroF1 = self.macroF1(predictions, y) microK = self.microK(predictions, y) macroK = self.macroK(predictions, y) - self.log('train-loss', loss, on_step=True, on_epoch=True, prog_bar=False, logger=True) - self.log('train-macroF1', macroF1, on_step=True, on_epoch=True, prog_bar=False, logger=True) - self.log('train-microF1', microF1, on_step=True, on_epoch=True, prog_bar=False, logger=True) - self.log('train-macroK', macroK, on_step=True, on_epoch=True, prog_bar=False, logger=True) - self.log('train-microK', microK, on_step=True, on_epoch=True, prog_bar=False, logger=True) + self.log('train-loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True) + self.log('train-macroF1', macroF1, on_step=False, on_epoch=True, prog_bar=False, logger=True) + self.log('train-microF1', microF1, on_step=False, on_epoch=True, prog_bar=False, logger=True) + self.log('train-macroK', macroK, on_step=False, on_epoch=True, prog_bar=False, logger=True) + self.log('train-microK', microK, on_step=False, on_epoch=True, prog_bar=False, logger=True) lX, ly = self._reconstruct_dict(predictions, y, batch_langs) return {'loss': loss, 'pred': lX, 'target': ly} @@ -96,6 +109,12 @@ class BertModel(pl.LightningModule): self.logger.experiment.add_scalars('train-langs-macroK', {f'{lang}': avg_macroK}, self.current_epoch) self.logger.experiment.add_scalars('train-langs-microK', {f'{lang}': avg_microK}, self.current_epoch) + if self.manual_log: + # Manual logging epoch loss + tr_epoch_loss = np.average([out['loss'].item() for out in outputs]) + self.csv_metrics['tr_loss'].append(tr_epoch_loss) + self.save_manual_logs() + def validation_step(self, val_batch, batch_idx): X, y, batch_langs = val_batch y = y.to('cuda' if self.gpus else 'cpu') @@ -106,11 +125,20 @@ class BertModel(pl.LightningModule): macroF1 = self.macroF1(predictions, y) microK = self.microK(predictions, y) macroK = self.macroK(predictions, y) - self.log('val-loss', loss, on_step=False, on_epoch=True, prog_bar=False, logger=True) + self.log('val-loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-macroF1', macroF1, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-microF1', microF1, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-macroK', macroK, on_step=False, on_epoch=True, prog_bar=True, logger=True) self.log('val-microK', microK, on_step=False, on_epoch=True, prog_bar=True, logger=True) + + if self.manual_log: + # Manual logging to csv + self.csv_metrics['va_loss'].append(loss.item()) + self.csv_metrics['va_macroF1'].append(macroF1.item()) + self.csv_metrics['va_microF1'].append(microF1.item()) + self.csv_metrics['va_macroK'].append(macroK.item()) + self.csv_metrics['va_microK'].append(microK.item()) + return {'loss': loss} def test_step(self, test_batch, batch_idx): @@ -126,8 +154,8 @@ class BertModel(pl.LightningModule): macroK = self.macroK(predictions, y) self.log('test-macroF1', macroF1, on_step=False, on_epoch=True, prog_bar=False, logger=True) self.log('test-microF1', microF1, on_step=False, on_epoch=True, prog_bar=False, logger=True) - self.log('test-macroK', macroK, on_step=False, on_epoch=True, prog_bar=True, logger=True) - self.log('test-microK', microK, on_step=False, on_epoch=True, prog_bar=True, logger=True) + self.log('test-macroK', macroK, on_step=False, on_epoch=True, prog_bar=False, logger=True) + self.log('test-microK', microK, on_step=False, on_epoch=True, prog_bar=False, logger=True) return def configure_optimizers(self, lr=1e-5, weight_decay=0.01): @@ -141,7 +169,7 @@ class BertModel(pl.LightningModule): 'weight_decay': weight_decay} ] optimizer = AdamW(optimizer_grouped_parameters, lr=lr) - scheduler = {'scheduler': StepLR(optimizer, step_size=25, gamma=1.0), # TODO set to 1.0 to debug (prev. 0.1) + scheduler = {'scheduler': StepLR(optimizer, step_size=25, gamma=0.1), 'interval': 'epoch'} return [optimizer], [scheduler] @@ -181,3 +209,14 @@ class BertModel(pl.LightningModule): for k, v in reconstructed_y.items(): reconstructed_y[k] = torch.cat(v).view(-1, predictions.shape[1]) return reconstructed_x, reconstructed_y + + def save_manual_logs(self): + if self.global_step == 0: + return + with open(self.csv_file, 'a', newline='\n') as handler: + writer = csv.writer(handler, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) + writer.writerow([np.average(self.csv_metrics['tr_loss']), np.average(self.csv_metrics['va_loss']), + np.average(self.csv_metrics['va_macroF1']), np.average(self.csv_metrics['va_microF1']), + np.average(self.csv_metrics['va_macroK']), np.average(self.csv_metrics['va_microK'])]) + + diff --git a/src/view_generators.py b/src/view_generators.py index 663e89d..858f751 100644 --- a/src/view_generators.py +++ b/src/view_generators.py @@ -450,7 +450,7 @@ class BertGen(ViewGen): self.patience = patience self.logger = TensorBoardLogger(save_dir='tb_logs', name='bert', default_hp_metric=False) self.early_stop_callback = EarlyStopping(monitor='val-macroF1', min_delta=0.00, - patience=self.patience, verbose=False, mode='max') + patience=self.patience, verbose=True, mode='max') # Zero shot parameters self.zero_shot = zero_shot @@ -475,14 +475,17 @@ class BertGen(ViewGen): create_if_not_exist(self.logger.save_dir) self.multilingualIndex.train_val_split(val_prop=0.2, max_val=2000, seed=1) bertDataModule = BertDataModule(self.multilingualIndex, batchsize=self.batch_size, max_len=512, - zero_shot=self.zero_shot, zscl_langs=self.train_langs) + zero_shot=self.zero_shot, zscl_langs=self.train_langs, + debug=True, max_samples=50) # todo: debug=True -> DEBUG setting if self.zero_shot: print(f'# Zero-shot setting! Training langs will be set to: {sorted(self.train_langs)}') - trainer = Trainer(gradient_clip_val=1e-1, max_epochs=self.nepochs, gpus=self.gpus, - logger=self.logger, callbacks=[self.early_stop_callback], checkpoint_callback=False, - overfit_batches=0.01) # todo: overfit_batches -> DEBUG setting + trainer = Trainer(max_epochs=self.nepochs, gpus=self.gpus, + logger=self.logger, + callbacks=[self.early_stop_callback], + checkpoint_callback=False) + trainer.fit(self.model, datamodule=bertDataModule) trainer.test(self.model, datamodule=bertDataModule) return self