.gitignore

@@ -182,10 +182,4 @@ scripts/
 logger/*
 explore_data.ipynb
 run.sh
-wandb/*
-hf_models
-embeddings/*
-results/*
-net.py
-stats.py
-plotters/
+wandb

amazon_categories.txt

@@ -0,0 +1,21 @@
+Appliances
+Arts Crafts and Sewing
+Automotive
+CDs and Vinyl
+Cell Phones and Accessories
+Electronics
+Grocery and Gourmet Food
+Home and Kitchen
+Industrial and Scientific
+Luxury Beauty
+Magazine Subscriptions
+Movies and TV
+Musical Instruments
+Office Products
+Patio Lawn and Garden
+Pet Supplies
+Software
+Sports and Outdoors
+Tools and Home Improvement
+Toys and Games
+Video Games

dataManager/clsDataset.py

@@ -1,6 +1,5 @@
 import sys
 import os
-import xml.etree.ElementTree as ET
 
 sys.path.append(os.getcwd())
 
@@ -9,90 +8,13 @@ import re
 from dataManager.multilingualDataset import MultilingualDataset
 
 CLS_PROCESSED_DATA_DIR = os.path.expanduser("~/datasets/cls-acl10-processed/")
-CLS_UNPROCESSED_DATA_DIR = os.path.expanduser("~/datasets/cls-acl10-unprocessed/")
-# LANGS = ["de", "en", "fr", "jp"]
-LANGS = ["de", "en", "fr"]
+LANGS = ["de", "en", "fr", "jp"]
 DOMAINS = ["books", "dvd", "music"]
 
 regex = r":\d+"
 subst = ""
 
 
-def load_unprocessed_cls(reduce_target_space=False):
-    data = {}
-    data_tr = []
-    data_te = []
-    c_tr = 0
-    c_te = 0
-    for lang in LANGS:
-        data[lang] = {}
-        for domain in DOMAINS:
-            data[lang][domain] = {}
-            print(f"lang: {lang}, domain: {domain}")
-            for split in ["train", "test"]:
-                domain_data = []
-                fdir = os.path.join(
-                    CLS_UNPROCESSED_DATA_DIR, lang, domain, f"{split}.review"
-                )
-                tree = ET.parse(fdir)
-                root = tree.getroot()
-                for child in root:
-                    if reduce_target_space:
-                        rating = np.zeros(3, dtype=int)
-                        original_rating = int(float(child.find("rating").text))
-                        # if original_rating < 3:
-                        if original_rating < 2:
-                            new_rating = 1
-                        # elif original_rating > 3:
-                        elif original_rating > 4:
-                            new_rating = 3
-                        else:
-                            new_rating = 2
-                        rating[new_rating - 1] = 1
-                        # rating = new_rating
-                    else:
-                        rating = np.zeros(5, dtype=int)
-                        rating[int(float(child.find("rating").text)) - 1] = 1
-                        # rating = new_rating
-                    # if split == "train":
-                    #     target_data = data_tr
-                    #     current_count = len(target_data)
-                    #     c_tr = +1
-                    # else:
-                    #     target_data = data_te
-                    #     current_count = len(target_data)
-                    #     c_te = +1
-                    domain_data.append(
-                    # target_data.append(
-                        {
-                            "asin": child.find("asin").text
-                            if child.find("asin") is not None
-                            else None,
-                            # "category": child.find("category").text
-                            # if child.find("category") is not None
-                            # else None,
-                            "category": domain,
-                            # "rating": child.find("rating").text
-                            # if child.find("rating") is not None
-                            # else None,
-                            "original_rating": int(float(child.find("rating").text)),
-                            "rating": rating.argmax(),
-                            "title": child.find("title").text
-                            if child.find("title") is not None
-                            else None,
-                            "text": child.find("text").text
-                            if child.find("text") is not None
-                            else None,
-                            "summary": child.find("summary").text
-                            if child.find("summary") is not None
-                            else None,
-                            "lang": lang,
-                        }
-                    )
-                data[lang][domain].update({split: domain_data})
-    return data
-
-
 def load_cls():
     data = {}
     for lang in LANGS:
@@ -102,7 +24,7 @@ def load_cls():
             train = (
                 open(
                     os.path.join(
-                        CLS_UNPROCESSED_DATA_DIR, lang, domain, "train.processed"
+                        CLS_PROCESSED_DATA_DIR, lang, domain, "train.processed"
                     ),
                     "r",
                 )
@@ -112,7 +34,7 @@ def load_cls():
             test = (
                 open(
                     os.path.join(
-                        CLS_UNPROCESSED_DATA_DIR, lang, domain, "test.processed"
+                        CLS_PROCESSED_DATA_DIR, lang, domain, "test.processed"
                     ),
                     "r",
                 )
@@ -137,33 +59,18 @@ def process_data(line):
 
 
 if __name__ == "__main__":
-    print(f"datapath: {CLS_UNPROCESSED_DATA_DIR}")
-    # data = load_cls()
-    data = load_unprocessed_cls(reduce_target_space=True)
-    multilingualDataset = MultilingualDataset(dataset_name="webis-cls-unprocessed")
-
+    print(f"datapath: {CLS_PROCESSED_DATA_DIR}")
+    data = load_cls()
+    multilingualDataset = MultilingualDataset(dataset_name="cls")
     for lang in LANGS:
-        # Xtr = [text["summary"] for text in data[lang]["books"]["train"]]
-        Xtr = [text["text"] for text in data[lang]["books"]["train"]]
-        Xtr += [text["text"] for text in data[lang]["dvd"]["train"]]
-        Xtr += [text["text"] for text in data[lang]["music"]["train"]]
+        # TODO: just using book domain atm
+        Xtr = [text[0] for text in data[lang]["books"]["train"]]
+        # Ytr = np.expand_dims([text[1] for text in data[lang]["books"]["train"]], axis=1)
+        Ytr = np.vstack([text[1] for text in data[lang]["books"]["train"]])
 
-        Ytr =[text["rating"] for text in data[lang]["books"]["train"]]
-        Ytr += [text["rating"] for text in data[lang]["dvd"]["train"]]
-        Ytr += [text["rating"] for text in data[lang]["music"]["train"]]
-
-        Ytr = np.vstack(Ytr)
-
-        Xte = [text["text"] for text in data[lang]["books"]["test"]]
-        Xte += [text["text"] for text in data[lang]["dvd"]["test"]]
-        Xte += [text["text"] for text in data[lang]["music"]["test"]]
-
-
-        Yte = [text["rating"] for text in data[lang]["books"]["test"]]
-        Yte += [text["rating"] for text in data[lang]["dvd"]["test"]]
-        Yte += [text["rating"] for text in data[lang]["music"]["test"]]
-
-        Yte = np.vstack(Yte)
+        Xte = [text[0] for text in data[lang]["books"]["test"]]
+        # Yte = np.expand_dims([text[1] for text in data[lang]["books"]["test"]], axis=1)
+        Yte = np.vstack([text[1] for text in data[lang]["books"]["test"]])
 
         multilingualDataset.add(
             lang=lang,
@@ -174,8 +81,6 @@ if __name__ == "__main__":
             tr_ids=None,
             te_ids=None,
         )
-    # multilingualDataset.save(
-    #     os.path.expanduser(
-    #         "~/datasets/cls-acl10-unprocessed/cls-acl10-unprocessed-all.pkl"
-    #     )
-    # )
+    multilingualDataset.save(
+        os.path.expanduser("~/datasets/cls-acl10-processed/cls-acl10-processed.pkl")
+    )

dataManager/gFunDataset.py

@@ -1,193 +1,219 @@
-import sys
 import os
-sys.path.append(os.path.expanduser("~/devel/gfun_multimodal"))
 
-from collections import defaultdict, Counter
-
-import numpy as np
-import re
-from tqdm import tqdm
-import pandas as pd
-from sklearn.model_selection import train_test_split
-
-# from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer
-# from dataManager.glamiDataset import get_dataframe
-# from dataManager.multilingualDataset import MultilingualDataset
+from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer
+from dataManager.glamiDataset import get_dataframe
+from dataManager.multilingualDataset import MultilingualDataset
 
 
-class SimpleGfunDataset:
+class gFunDataset:
     def __init__(
         self,
-            dataset_name=None,
-            datapath="~/datasets/rai/csv/test.csv",
-            textual=True,
-            visual=False,
-            multilabel=False,
-            set_tr_langs=None,
-            set_te_langs=None,
-            reduced=False,
-            only_inference=False,
-            labels=None
+        dataset_dir,
+        is_textual,
+        is_visual,
+        is_multilabel,
+        labels=None,
+        nrows=None,
+        data_langs=None,
     ):
-        self.name = dataset_name
-        self.datadir = os.path.expanduser(datapath)
-        self.textual = textual
-        self.visual = visual
-        self.multilabel = multilabel
-        self.reduced = reduced
-        self.only_inference = only_inference
+        self.dataset_dir = dataset_dir
+        self.data_langs = data_langs
+        self.is_textual = is_textual
+        self.is_visual = is_visual
+        self.is_multilabel = is_multilabel
         self.labels = labels
-        self.load_csv(set_tr_langs, set_te_langs)
-        self.print_stats()
+        self.nrows = nrows
+        self.dataset = {}
+        self._load_dataset()
 
-    def print_stats(self):
-        print(f"Dataset statistics\n{'-' * 15}")
-        tr = 0
-        va = 0
-        te = 0
-        for lang in self.all_langs:
-            n_tr = len(self.train_data[lang]) if lang in self.tr_langs and self.train_data is not None else 0
-            n_va = len(self.val_data[lang]) if lang in self.tr_langs and self.val_data is not None else 0 
-            n_te = len(self.test_data[lang])
-            tr += n_tr
-            va += n_va
-            te += n_te
-            print(f"{lang} - tr: {n_tr} - va: {n_va} - te: {n_te}")
-        print(f"{'-' * 15}\nTotal\n{'-' * 15}")
-        print(f"tr: {tr} - va: {va} - te: {te}")
-    
-    def load_csv_inference(self):
-        test = pd.read_csv(self.datadir)
-        self._set_labels(test)
-        self._set_langs(train=None, test=test)
-        self.train_data = None
-        self.val_data = None
-        self.test_data = self._set_datalang(test)
-        return 
-
-    def load_csv(self, set_tr_langs, set_te_langs):
-        if self.only_inference:
-            self.load_csv_inference()
-            return
-        _data_tr = pd.read_csv(os.path.join(self.datadir, "train.csv" if not self.reduced else "train.small.csv"))
-        try:
-            stratified = "class"
-            train, val = train_test_split(_data_tr, test_size=0.2, random_state=42, stratify=_data_tr.label)
-        except:
-            stratified = "lang"
-            train, val = train_test_split(_data_tr, test_size=0.2, random_state=42, stratify=_data_tr.lang)
-        print(f"- dataset stratified by {stratified}")
-        test = pd.read_csv(os.path.join(self.datadir, "test.small.csv" if not self.reduced else "test.small.csv"))
-        self._set_langs (train, test, set_tr_langs, set_te_langs)
-        self._set_labels(_data_tr)
-        self.full_train = _data_tr
-        self.full_test = self.test
-        self.train_data = self._set_datalang(train)
-        self.val_data = self._set_datalang(val)
-        self.test_data = self._set_datalang(test)
-        return
-
-    def _set_labels(self, data):
-        if self.labels is not None:
-            self.labels = [i for i in range(self.labels)]
+    def get_label_binarizer(self, labels):
+        if self.dataset_name in ["rcv1-2", "jrc", "cls"]:
+            mlb = "Labels are already binarized for rcv1-2 dataset"
+        elif self.is_multilabel:
+            mlb = MultiLabelBinarizer()
+            mlb.fit([labels])
         else:
-            self.labels = sorted(list(data.label.unique()))
+            mlb = LabelBinarizer()
+            mlb.fit(labels)
+        return mlb
 
-    def _set_langs(self, train, test, set_tr_langs=None, set_te_langs=None):
-        self.tr_langs = set(train.lang.unique().tolist()) if train is not None else set()
-        self.te_langs = set(test.lang.unique().tolist()) if test is not None else set()
-        if set_tr_langs is not None:
-            print(f"-- [SETTING TRAINING LANGS TO: {list(set_tr_langs)}]")
-            self.tr_langs = self.tr_langs.intersection(set(set_tr_langs))
-        if set_te_langs is not None:
-            print(f"-- [SETTING TESTING LANGS TO: {list(set_tr_langs)}]")
-            self.te_langs = self.te_langs.intersection(set(set_te_langs))
-        self.all_langs = self.tr_langs.union(self.te_langs)
-    
-        return self.tr_langs, self.te_langs, self.all_langs
-    
-    def _set_datalang(self, data: pd.DataFrame):
-        return {lang: data[data.lang == lang] for lang in self.all_langs}
-
-    def training(self, merge_validation=False, mask_number=False, target_as_csr=False):
-        apply_mask = lambda x: _mask_numbers(x) if _mask_numbers else x
-        lXtr = {
-            lang: {"text": apply_mask(self.train_data[lang].text.tolist())}
-            for lang in self.tr_langs
-            }
-        if merge_validation:
-            for lang in self.tr_langs:
-                lXtr[lang]["text"] += apply_mask(self.val_data[lang].text.tolist())
-
-        lYtr = {
-            lang: self.train_data[lang].label.tolist() for lang in self.tr_langs
-        }
-        if merge_validation:
-            for lang in self.tr_langs:
-                lYtr[lang] += self.val_data[lang].label.tolist()
-
-        for lang in self.tr_langs:
-            lYtr[lang] = self.indices_to_one_hot(
-                indices = lYtr[lang],
-                n_labels = self.num_labels()
+    def _load_dataset(self):
+        if "glami" in self.dataset_dir.lower():
+            print(f"- Loading GLAMI dataset from {self.dataset_dir}")
+            self.dataset_name = "glami"
+            self.dataset, self.labels, self.data_langs = self._load_glami(
+                self.dataset_dir, self.nrows
             )
+            self.mlb = self.get_label_binarizer(self.labels)
+
+        elif "rcv" in self.dataset_dir.lower():
+            print(f"- Loading RCV1-2 dataset from {self.dataset_dir}")
+            self.dataset_name = "rcv1-2"
+            self.dataset, self.labels, self.data_langs = self._load_multilingual(
+                self.dataset_name, self.dataset_dir, self.nrows
+            )
+            self.mlb = self.get_label_binarizer(self.labels)
+
+        elif "jrc" in self.dataset_dir.lower():
+            print(f"- Loading JRC dataset from {self.dataset_dir}")
+            self.dataset_name = "jrc"
+            self.dataset, self.labels, self.data_langs = self._load_multilingual(
+                self.dataset_name, self.dataset_dir, self.nrows
+            )
+            self.mlb = self.get_label_binarizer(self.labels)
+
+        elif "cls" in self.dataset_dir.lower():
+            print(f"- Loading CLS dataset from {self.dataset_dir}")
+            self.dataset_name = "cls"
+            self.dataset, self.labels, self.data_langs = self._load_multilingual(
+                self.dataset_name, self.dataset_dir, self.nrows
+            )
+            self.mlb = self.get_label_binarizer(self.labels)
+
+        self.show_dimension()
+
+        return
+
+    def show_dimension(self):
+        print(f"\n[Dataset: {self.dataset_name.upper()}]")
+        for lang, data in self.dataset.items():
+            print(
+                f"-- Lang: {lang} - train docs: {len(data['train']['text'])} - test docs: {len(data['test']['text'])}"
+            )
+        if self.dataset_name in ["rcv1-2", "jrc", "cls"]:
+            print(f"-- Labels: {self.labels}")
+        else:
+            print(f"-- Labels: {len(self.labels)}")
+
+    def _load_multilingual(self, dataset_name, dataset_dir, nrows):
+        old_dataset = MultilingualDataset(dataset_name=dataset_name).load(dataset_dir)
+        if nrows is not None:
+            if dataset_name == "cls":
+                old_dataset.reduce_data(langs=["de", "en", "fr"], maxn=nrows)
+            else:
+                old_dataset.reduce_data(langs=["en", "it", "fr"], maxn=nrows)
+        labels = old_dataset.num_labels()
+        data_langs = old_dataset.langs()
+
+        def _format_multilingual(data):
+            text = data[0]
+            image = None
+            labels = data[1]
+            return {"text": text, "image": image, "label": labels}
+
+        dataset = {
+            k: {"train": _format_multilingual(v[0]), "test": _format_multilingual(v[1])}
+            for k, v in old_dataset.multiling_dataset.items()
+        }
+        return dataset, labels, data_langs
+
+    def _load_glami(self, dataset_dir, nrows):
+        train_split = get_dataframe("train", dataset_dir=dataset_dir).sample(n=nrows)
+        test_split = get_dataframe("test", dataset_dir=dataset_dir).sample(
+            n=int(nrows / 10)
+        )
+
+        gb_train = train_split.groupby("geo")
+        gb_test = test_split.groupby("geo")
+
+        if self.data_langs is None:
+            data_langs = sorted(train_split.geo.unique().tolist())
+        if self.labels is None:
+            labels = train_split.category_name.unique().tolist()
+
+        def _format_glami(data_df):
+            text = (data_df.name + " " + data_df.description).tolist()
+            image = data_df.image_file.tolist()
+            labels = data_df.category_name.tolist()
+            return {"text": text, "image": image, "label": labels}
+
+        dataset = {
+            lang: {
+                "train": _format_glami(data_tr),
+                "test": _format_glami(gb_test.get_group(lang)),
+            }
+            for lang, data_tr in gb_train
+            if lang in data_langs
+        }
+
+        return dataset, labels, data_langs
+
+    def binarize_labels(self, labels):
+        if self.dataset_name in ["rcv1-2", "jrc", "cls"]:
+            # labels are already binarized for rcv1-2 dataset
+            return labels
+        if hasattr(self, "mlb"):
+            return self.mlb.transform(labels)
+        else:
+            raise AttributeError("Label binarizer not found")
+
+    def training(self):
+        lXtr = {}
+        lYtr = {}
+        for lang in self.data_langs:
+            text = self.dataset[lang]["train"]["text"] if self.is_textual else None
+            img = self.dataset[lang]["train"]["image"] if self.is_visual else None
+            labels = self.dataset[lang]["train"]["label"]
+
+            lXtr[lang] = {"text": text, "image": img}
+            lYtr[lang] = self.binarize_labels(labels)
 
         return lXtr, lYtr
 
-    def test(self, mask_number=False, target_as_csr=False):
-        apply_mask = lambda x: _mask_numbers(x) if _mask_numbers else x
-        lXte = {
-            lang: {
-                "text": apply_mask(self.test_data[lang].text.tolist()),
-                "id": self.test_data[lang].id.tolist()
-                }
-            for lang in self.te_langs
-            }
-        if not self.only_inference:
-            lYte = {
-                lang: self.indices_to_one_hot(
-                    indices=self.test_data[lang].label.tolist(),
-                    n_labels=self.num_labels())
-                for lang in self.te_langs
-            }
-        else:
-            lYte = None
+    def test(self):
+        lXte = {}
+        lYte = {}
+        for lang in self.data_langs:
+            text = self.dataset[lang]["test"]["text"] if self.is_textual else None
+            img = self.dataset[lang]["test"]["image"] if self.is_visual else None
+            labels = self.dataset[lang]["test"]["label"]
+
+            lXte[lang] = {"text": text, "image": img}
+            lYte[lang] = self.binarize_labels(labels)
+
         return lXte, lYte
 
     def langs(self):
-        return list(self.all_langs)
+        return self.data_langs
 
     def num_labels(self):
+        if self.dataset_name not in ["rcv1-2", "jrc", "cls"]:
             return len(self.labels)
+        else:
+            return self.labels
 
-    def indices_to_one_hot(self, indices, n_labels):
-        one_hot_matrix = np.zeros((len(indices), n_labels))
-        one_hot_matrix[np.arange(len(indices)), indices] = 1
-        return one_hot_matrix
+    def save_as_pickle(self, path):
+        import pickle
 
-    def get_name(self):
-        return self.name
+        filepath = os.path.join(path, f"{self.dataset_name}_{self.nrows}.pkl")
+        with open(filepath, "wb") as f:
+            print(f"- saving dataset in {filepath}")
+            pickle.dump(self, f)
 
 
-def _mask_numbers(data):
-    mask_moredigit = re.compile(r"\s[\+-]?\d{5,}([\.,]\d*)*\b")
-    mask_4digit = re.compile(r"\s[\+-]?\d{4}([\.,]\d*)*\b")
-    mask_3digit = re.compile(r"\s[\+-]?\d{3}([\.,]\d*)*\b")
-    mask_2digit = re.compile(r"\s[\+-]?\d{2}([\.,]\d*)*\b")
-    mask_1digit = re.compile(r"\s[\+-]?\d{1}([\.,]\d*)*\b")
-    masked = []
-    for text in tqdm(data, desc="masking numbers", disable=True):
-        text = " " + text
-        text = mask_moredigit.sub(" MoreDigitMask", text)
-        text = mask_4digit.sub(" FourDigitMask", text)
-        text = mask_3digit.sub(" ThreeDigitMask", text)
-        text = mask_2digit.sub(" TwoDigitMask", text)
-        text = mask_1digit.sub(" OneDigitMask", text)
-        masked.append(text.replace(".", "").replace(",", "").strip())
-    return masked
-
 if __name__ == "__main__":
-    data_rai = SimpleGfunDataset()
-    lXtr, lYtr = data_rai.training(mask_number=False)
-    lXte, lYte = data_rai.test(mask_number=False)
-    exit()
+    import os
+
+    GLAMI_DATAPATH = os.path.expanduser("~/datasets/GLAMI-1M-dataset")
+    RCV_DATAPATH = os.path.expanduser(
+        "~/datasets/rcv1-2/rcv1-2_doclist_trByLang1000_teByLang1000_processed_run0.pickle"
+    )
+    JRC_DATAPATH = os.path.expanduser(
+        "~/datasets/jrc/jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.pickle"
+    )
+
+    print("Hello gFunDataset")
+    dataset = gFunDataset(
+        # dataset_dir=GLAMI_DATAPATH,
+        # dataset_dir=RCV_DATAPATH,
+        dataset_dir=JRC_DATAPATH,
+        data_langs=None,
+        is_textual=True,
+        is_visual=True,
+        is_multilabel=False,
+        labels=None,
+        nrows=13,
+    )
+    lXtr, lYtr = dataset.training()
+    lXte, lYte = dataset.test()
+    exit(0)

dataManager/multilingualDataset.py

@@ -222,37 +222,6 @@ class MultilingualDataset:
                 new_data.append((docs[:maxn], labels[:maxn], None))
         return new_data
 
-    def from_csv(self, path_tr, path_te):
-        import pandas as pd
-        from os.path import expanduser
-        train = pd.read_csv(expanduser(path_tr))
-        test = pd.read_csv(expanduser(path_te))
-        for lang in train.lang.unique():
-            tr_datalang = train.loc[train["lang"] == lang]
-            Xtr = tr_datalang.text.to_list()
-            tr_labels = tr_datalang.label.to_list()
-            Ytr = np.zeros((len(Xtr), 28), dtype=int)
-            for j, i in enumerate(tr_labels):
-                Ytr[j, i] = 1
-            tr_ids = tr_datalang.id.to_list()
-            te_datalang = test.loc[test["lang"] == lang]
-            Xte = te_datalang.text.to_list()
-            te_labels = te_datalang.label.to_list()
-            Yte = np.zeros((len(Xte), 28), dtype=int)
-            for j, i in enumerate(te_labels):
-                Yte[j, i] = 1
-            te_ids = te_datalang.id.to_list()
-            self.add(
-                lang=lang,
-                Xtr=Xtr,
-                Ytr=Ytr,
-                Xte=Xte,
-                Yte=Yte,
-                tr_ids=tr_ids,
-                te_ids=te_ids
-            )
-        return self
-
 
 def _mask_numbers(data):
     mask_moredigit = re.compile(r"\s[\+-]?\d{5,}([\.,]\d*)*\b")
@@ -271,6 +240,7 @@ def _mask_numbers(data):
         masked.append(text.replace(".", "").replace(",", "").strip())
     return masked
 
+
 if __name__ == "__main__":
     DATAPATH = expanduser(
         "~/datasets/rcv1-2/rcv1-2_doclist_trByLang1000_teByLang1000_processed_run0.pickle"

dataManager/utils.py

@@ -1,5 +1,7 @@
 from os.path import expanduser, join
-from dataManager.gFunDataset import SimpleGfunDataset
+from dataManager.gFunDataset import gFunDataset
+from dataManager.multiNewsDataset import MultiNewsDataset
+from dataManager.amazonDataset import AmazonDataset
 
 
 def load_from_pickle(path, dataset_name, nrows):
@@ -14,15 +16,81 @@ def load_from_pickle(path, dataset_name, nrows):
     return loaded
 
 
-def get_dataset(dataset_path, args):
-    dataset = SimpleGfunDataset(
-        dataset_name="rai",
-        datapath=dataset_path,
-        textual=True,
-        visual=False,
-        multilabel=False,
-        set_tr_langs=args.tr_langs,
-        set_te_langs=args.te_langs,
-        reduced=args.reduced
+def get_dataset(dataset_name, args):
+    assert dataset_name in [
+        "multinews",
+        "amazon",
+        "rcv1-2",
+        "glami",
+        "cls",
+    ], "dataset not supported"
+
+    RCV_DATAPATH = expanduser(
+        "~/datasets/rcv1-2/rcv1-2_doclist_trByLang1000_teByLang1000_processed_run0.pickle"
     )
+    JRC_DATAPATH = expanduser(
+        "~/datasets/jrc/jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.pickle"
+    )
+    CLS_DATAPATH = expanduser("~/datasets/cls-acl10-processed/cls-acl10-processed.pkl")
+
+    MULTINEWS_DATAPATH = expanduser("~/datasets/MultiNews/20110730/")
+
+    GLAMI_DATAPATH = expanduser("~/datasets/GLAMI-1M-dataset")
+
+    if dataset_name == "multinews":
+        # TODO: convert to gFunDataset
+        raise NotImplementedError
+        dataset = MultiNewsDataset(
+            expanduser(MULTINEWS_DATAPATH),
+            excluded_langs=["ar", "pe", "pl", "tr", "ua"],
+        )
+    elif dataset_name == "amazon":
+        # TODO: convert to gFunDataset
+        raise NotImplementedError
+        dataset = AmazonDataset(
+            domains=args.domains,
+            nrows=args.nrows,
+            min_count=args.min_count,
+            max_labels=args.max_labels,
+        )
+    elif dataset_name == "jrc":
+        dataset = gFunDataset(
+            dataset_dir=JRC_DATAPATH,
+            is_textual=True,
+            is_visual=False,
+            is_multilabel=True,
+            nrows=args.nrows,
+        )
+    elif dataset_name == "rcv1-2":
+        dataset = gFunDataset(
+            dataset_dir=RCV_DATAPATH,
+            is_textual=True,
+            is_visual=False,
+            is_multilabel=True,
+            nrows=args.nrows,
+        )
+    elif dataset_name == "glami":
+        if args.save_dataset is False:
+            dataset = load_from_pickle(GLAMI_DATAPATH, dataset_name, args.nrows)
+        else:
+            dataset = gFunDataset(
+                dataset_dir=GLAMI_DATAPATH,
+                is_textual=True,
+                is_visual=True,
+                is_multilabel=False,
+                nrows=args.nrows,
+            )
+
+            dataset.save_as_pickle(GLAMI_DATAPATH)
+
+    elif dataset_name == "cls":
+        dataset = gFunDataset(
+            dataset_dir=CLS_DATAPATH,
+            is_textual=True,
+            is_visual=False,
+            is_multilabel=False,
+            nrows=args.nrows,
+        )
+    else:
+        raise NotImplementedError
     return dataset

evaluation/compute_results.py

@@ -1,52 +0,0 @@
-from argparse import ArgumentParser
-from csvlogger import CsvLogger
-import pandas as pd
-from sklearn.metrics import mean_absolute_error
-
-from os.path import join
-
-
-def main():
-    # SETTINGS = ["p", "m", "w", "t", "mp", "mpw", "mpt", "mptw"]
-    SETTINGS = ["mbert"]
-    results = []
-    for setting in SETTINGS:
-        results.append(evalaute(setting))
-    df = pd.DataFrame()
-    for r in results:
-        df = df.append(r)
-    print(df)
-
-
-def evalaute(setting, result_file="preds.gfun.csv"):
-    result_dir = "results"
-    print(f"- reading from: {result_file}")
-    df = pd.read_csv(join(result_dir, result_file))
-    langs = df.langs.unique()
-    res = []
-    for lang in langs:
-        l_df = df.langs == lang
-        selected_neg = df.labels == 0
-        seleteced_neutral = df.labels == 1
-        selected_pos = df.labels == 2
-        neg = df[l_df & selected_neg]
-        neutral = df[l_df & seleteced_neutral]
-        pos = df[l_df & selected_pos]
-
-        # print(f"{lang=}")
-        # print(neg.shape, neutral.shape, pos.shape)
-
-        neg_mae = mean_absolute_error(neg.labels, neg.preds).round(3)
-        neutral_mae = mean_absolute_error(neutral.labels, neutral.preds).round(3)
-        pos_mae = mean_absolute_error(pos.labels, pos.preds).round(3)
-
-        macro_mae = ((neg_mae + neutral_mae + pos_mae) / 3).round(3)
-        # print(f"{lang=} - {neg_mae=}, {neutral_mae=}, {pos_mae=}, {macro_mae=}")
-        res.append([lang, neg_mae, neutral_mae, pos_mae, setting])
-    return res
-        
-
-
-
-if __name__ == "__main__":
-    main()

evaluation/evaluate.py

@@ -1,9 +1,8 @@
 from joblib import Parallel, delayed
 from collections import defaultdict
 
-# from evaluation.metrics import *
-import numpy as np
-from sklearn.metrics import accuracy_score, top_k_accuracy_score, f1_score, precision_score, recall_score
+from evaluation.metrics import *
+from sklearn.metrics import accuracy_score, top_k_accuracy_score, f1_score
 
 
 def evaluation_metrics(y, y_, clf_type):
@@ -14,17 +13,13 @@ def evaluation_metrics(y, y_, clf_type):
             # TODO: we need logits top_k_accuracy_score(y, y_, k=10),
             f1_score(y, y_, average="macro", zero_division=1),
             f1_score(y, y_, average="micro"),
-            precision_score(y, y_, zero_division=1, average="macro"),
-            recall_score(y, y_, zero_division=1, average="macro"),
         )
     elif clf_type == "multilabel":
         return (
-            f1_score(y, y_, average="macro", zero_division=1),
-            f1_score(y, y_, average="micro"),
-            0,
-            0,
-            # macroK(y, y_),
-            # microK(y, y_),
+            macroF1(y, y_),
+            microF1(y, y_),
+            macroK(y, y_),
+            microK(y, y_),
         )
     else:
         raise ValueError("clf_type must be either 'singlelabel' or 'multilabel'")
@@ -52,11 +47,9 @@ def log_eval(l_eval, phase="training", clf_type="multilabel", verbose=True):
     metrics = []
 
     if clf_type == "multilabel":
-        for lang in sorted(l_eval.keys()):
-            # macrof1, microf1, macrok, microk = l_eval[lang]
-            # metrics.append([macrof1, microf1, macrok, microk])
-            macrof1, microf1, precision, recall = l_eval[lang]
-            metrics.append([macrof1, microf1, precision, recall])
+        for lang in l_eval.keys():
+            macrof1, microf1, macrok, microk = l_eval[lang]
+            metrics.append([macrof1, microf1, macrok, microk])
             if phase != "validation":
                 print(f"Lang {lang}: macro-F1 = {macrof1:.3f} micro-F1 = {microf1:.3f}")
         averages = np.mean(np.array(metrics), axis=0)
@@ -76,15 +69,12 @@ def log_eval(l_eval, phase="training", clf_type="multilabel", verbose=True):
             # "acc10",  # "accuracy-at-10",
             "MF1",  # "macro-F1",
             "mF1",  # "micro-F1",
-            "precision",
-            "recall"
         ]
-        for lang in sorted(l_eval.keys()):
+        for lang in l_eval.keys():
             # acc, top5, top10, macrof1, microf1 = l_eval[lang]
-            acc, macrof1, microf1, precision, recall= l_eval[lang]
+            acc, macrof1, microf1 = l_eval[lang]
             # metrics.append([acc, top5, top10, macrof1, microf1])
-            # metrics.append([acc, macrof1, microf1])
-            metrics.append([acc, macrof1, microf1, precision, recall])
+            metrics.append([acc, macrof1, microf1])
 
             for m, v in zip(_metrics, l_eval[lang]):
                 lang_metrics[m][lang] = v
@@ -92,8 +82,7 @@ def log_eval(l_eval, phase="training", clf_type="multilabel", verbose=True):
             if phase != "validation":
                 print(
                     # f"Lang {lang}: acc = {acc:.3f} acc-top5 = {top5:.3f} acc-top10 = {top10:.3f} macro-F1: {macrof1:.3f} micro-F1 = {microf1:.3f}"
-                    # f"Lang {lang}: acc = {acc:.3f} macro-F1: {macrof1:.3f} micro-F1 = {microf1:.3f}"
-                    f"Lang {lang}: acc = {acc:.3f} macro-F1: {macrof1:.3f} micro-F1 = {microf1:.3f} pr = {precision:.3f} re = {recall:.3f}"
+                    f"Lang {lang}: acc = {acc:.3f} macro-F1: {macrof1:.3f} micro-F1 = {microf1:.3f}"
                 )
         averages = np.mean(np.array(metrics), axis=0)
         if verbose:

example.sh

@@ -1,7 +0,0 @@
-#!bin/bash
-
-# carica il dataset da examples/dataset/sample-dataset.csv
-# le predicions vengono salvate in exampels/results/sample-dataset_<timestamp>.csv
-# --category_map specifica il path del file di mapping. Nel file di output sono salvate i nomi delle categorie predette.
-
-python infer.py --datapath examples/dataset/sample-dataset.csv --outdir examples/results --category_map examples/dataset/dataset-mapping.csv

examples/dataset/dataset-mapping.csv

@@ -1,29 +0,0 @@
-category,id
-agricoltura-zootecnia,0
-ambiente-natura-territorio,1
-arte-artigianato,2
-avvenimenti-celebrazioni-eventi_storici,3
-beni_culturali,4
-commercio,5
-consumi-servizi,6
-cronaca,7
-cultura-scienze_umane,8
-economia-credito-finanza,9
-editoria-stampa-mass_media,10
-esteri,11
-giustizia-criminalita-sicurezza,12
-individuo-famiglia-associazioni-societa,13
-industria-impresa-produzione,14
-istruzione-formazione,15
-lavoro-previdenza,16
-musica_e_spettacolo,17
-non_classificabile,18
-politica-partiti-istituzioni-sindacati,19
-pubblica_amministrazione-enti_locali,20
-sanita-salute,21
-scienze-tecnologie,22
-sport,23
-tempo_libero,24
-trasporti,25
-usi_e_costumi,26
-vita_e_cultura_religiosa,27

examples/dataset/sample-dataset.csv

@@ -1,10 +0,0 @@
-id,lang,provider,date,title,text,label,str_label
-it_74231,it,,,," - CITTA' DEL VATICANO, 18 FEB - Laboratori, video, relazioni e confronti. Da giovedì a domenica in Vaticano si troveranno faccia a faccia i presidenti delle conferenze episcopali di tutto il mondo per affrontare il tema degli abusi sessuali e della pedofilia nella chiesa nell'incontro ""La protezione dei minori nella Chiesa"". L'evento è stato presentato oggi nella sala stampa vaticana, alla presenza di padre Lombardi, che sarà il moderatore, di due componenti il comitato organizzativo, l'arcivescovo di Chicago, card. Blase Cupich, e quello di Malta, mons. Charles Scicluna, e del referente del comitato, padre Hans Zollner, già presidente del Centro per la protezione dei Minori della Pontificia Università Gregoriana e membro della Pontificia Commissione per la tutela dei minori. ""Dobbiamo fare tutto il possibile, ognuno nel proprio ruolo, affinché la chiesa sia un casa sicura e accogliente per i più deboli"", ha detto card. Cupich. Certificazione ISO 9001. I ""processi di Produzione, distribuzione e pubblicazione in formato multimediale di notizie giornalistiche"" ANSA sono certificati in conformità alla normativa internazionale UNI EN ISO 9001:2015. Politica per la Qualità",27,vita_e_cultura_religiosa
-it_39668,it,,,,"Gli operai della Blutec e dell'indotto hanno occupato il municipio di Termini Imerese. Protestano per il mancato rispetto degli impegni aziendali per il rilancio della fabbrica e per il timore che la cassa integrazione non sia prorogata per quest'anno. ""Gli operai sono disperati, - dice il sindaco Francesco Giunta - credo non si sia mai arrivati così in basso. Che il governo prenda posizione"". Certificazione ISO 9001. I ""processi di Produzione, distribuzione e pubblicazione in formato multimediale di notizie giornalistiche"" ANSA sono certificati in conformità alla normativa internazionale UNI EN ISO 9001:2015. Politica per la Qualità",14,industria-impresa-produzione
-it_74956,it,,,,"I vescovi spagnoli dicono di non poter negare una sepoltura cristiana a Francisco Franco nella cattedrale dell'Almudena, in cui la famiglia del dittatore ha una cripta",27,vita_e_cultura_religiosa
-it_46451,it,,,,"L’aula della Camera ha approvato in via definitiva la conversione in legge del decreto sul lavoro con 279 sì e 143 no. Con il voto di Montecitorio, è dunque diventato legge il decreto sul lavoro. Il nuovo testo modifica profondamente l’attuale normativa sull’apprendistato e i contratti a termine. Le disposizioni si applicano ai rapporti di lavoro costituiti successivamente alla data di entrata in vigore del decreto. Assi portanti sono le norme sui contratti a tempo e quelle sull’apprendistato. D’ora in poi, infatti, sarà possibile stipulare contratti fino a 36 mesi senza causale, vale a dire senza una ragione specifica, così come sarà possibile prorogare un contratto per cinque volte (non più otto come inizialmente previsto dal testo del governo). Fissato anche un tetto di precari pari al 20 per cento dell’organico stabile: in caso di violazione, il datore di lavoro si troverà a dover pagare una multa, ma non è più obbligato - come concordato inizialmente durante l’esame del dl alla Camera - a stabilizzare i lavoratori assunti fuori quota. Altra novità, il ripristino della formazione pubblica per gli apprendisti seppure con alcuni paletti, sulla maternità e sui contratti di solidarietà. Ma ecco i punti, in sintesi. La durata dei contratti a termine Il contratto a termine non può avere una durata superiore a trentasei mesi, comprensiva di eventuali proroghe, per lo svolgimento di qualunque tipo di mansione, sia nella forma del contratto a tempo determinato, sia nell’ambito di un contratto di somministrazione a tempo determinato. Tempo determinato Il numero complessivo di contratti a tempo determinato stipulati da ciascun datore di lavoro non può eccedere il limite del 20 per cento del numero dei lavoratori a tempo indeterminato in forza al 1 gennaio dell’anno di assunzione. Per i datori di lavoro che occupano fino a cinque dipendenti è sempre possibile stipulare un contratto di lavoro a tempo determinato. I lavoratori assunti a termine in violazione del limite percentuale sono considerati lavoratori subordinati con contratto a tempo indeterminato sin dalla data di costituzione del rapporto di lavoro. L’apprendistato Ferma restando la possibilità per i contratti collettivi nazionali di lavoro di individuare limiti diversi, esclusivamente per i datori di lavoro che occupano almeno trenta dipendenti, l’assunzione di nuovi apprendisti è subordinata alla prosecuzione, a tempo indeterminato, del rapporto di lavoro al termine del periodo di apprendistato, nei trentasei mesi precedenti la nuova assunzione, di almeno il 20 per cento degli apprendisti dipendenti dallo stesso datore di lavoro. Le proroghe Le proroghe sono ammesse, fino ad un massimo di cinque volte, nell’arco dei complessivi trentasei mesi, indipendentemente dal numero dei rinnovi. Nel computo del periodo massimo di durata del contratto a tempo determinato si tiene conto anche dei periodi di missione aventi ad oggetto mansioni equivalenti per la somministrazione di lavoro a tempo determinato. Viene prevista esplicitamente l’adozione in via sperimentale del contratto a protezione crescente nella futura delega legislativa che dovrà disciplinare, in un testo unico semplificato, i rapporti di lavoro. 15 maggio 2014 | 20:14 © RIPRODUZIONE RISERVATA",16,lavoro-previdenza
-it_14143,it,,,," - BOLZANO, 31 MAG - Tra novembre 2015 e aprile 2016 il turismo in Alto Adige ha registrato un +7,8% negli arrivi e un +6,2% nelle presenze rispetto alla stagione invernale precedente in tutti e dieci i consorzi turistici. Lo rileva l'Istituto provinciale di statistica Astat. Notevole è l'aumento delle presenze negli alloggi agrituristici, che registrano un +8,8%, pari a 58 mila presenze in più rispetto all'inverno precedente. In aumento gli ospiti italiani: +9,3% negli arrivi e +8,5% nelle presenze.",6,consumi-servizi
-it_60547,it,,,," - ROMA, 08 MAG - È già partita la realizzazione del prossimo supercomputer più veloce del mondo, in grado di compiere un numero di operazioni al secondo da capogiro: un miliardo e mezzo di miliardi. Vedrà la luce negli Stati uniti, al Laboratorio Nazionale di Oak Ridge (Ornl) del Dipartimento dell'Energia americano (Doe), e si prevede che sarà online nel 2021. Il supercomputer, chiamato Frontier, costerà oltre 600 milioni di dollari e si servirà di sistemi di elaborazione estremamente avanzati e dell'Intelligenza Artificiale. La potenza di calcolo senza precedenti di Frontier e le sue tecnologie di nuova generazione saranno un'incredibile risorsa per i ricercatori, che le utilizzeranno per far avanzare le conoscenze in tantissimi campi: dalle scienze del clima alla genomica, dalla fisica alle strutture sub-atomiche. ""Frontier rappresenta lo stato dell'arte nell'informatica ad alte prestazioni"", commenta Jeff Nichols, ricercatore all'Oak Ridge. Certificazione ISO 9001. I ""processi di Produzione, distribuzione e pubblicazione in formato multimediale di notizie giornalistiche"" ANSA sono certificati in conformità alla normativa internazionale UNI EN ISO 9001:2015. Politica per la Qualità",22,scienze-tecnologie
-it_24656,it,,,,Ai cronisti tre mensilità di stipendio e un contributo di 700 euro per l'affitto della casa (per massimo un anno). Fino a 18 mensilità per chi decidesse di interrompere il rapporto di lavoro,9,economia-credito-finanza
-it_27162,it,,,,"I prezzi dei carburanti dovrebbero registrare un aumento nei prossimi giorni. A sostenerlo è Bruno Bearzi, il presidente nazionale della Figisc - Confcommercio nell'Osservatorio prezzi. ""A meno di drastiche variazioni in più od in meno delle quotazioni internazionali alla chiusura dei mercati di oggi o del tasso di cambio euro/dollaro - sottolinea Bearzi - ci sono ad oggi plausibili presupposti per una aspettativa di prezzi tendenzialmente in aumento, media dei due prodotti benzina e gasolio e delle due modalità di servizio 'self' e 'servito', per i prossimi 4 giorni con scostamenti, in questa prima fase iniziale, entro 0,5 cent al litro in più"". Al monitoraggio, effettuato in collaborazione con Assopetroli - Assoenergia, ""dei prezzi pubblicati dalla Commissione Ue risulta che nella data del 21 ottobre lo 'stacco Italia delle imposte sui carburanti' (ovvero quante imposte si pagano in più in Italia rispetto alla media dei 28 Paesi Ue) è pari a +22,4 cent/litro per la benzina e +20,4 per il gasolio (in media ponderale tra i prodotti +21,1) e le imposte hanno inciso nella settimana sul prezzo finale della benzina per il 63,04% e per il 58,34% su quello del gasolio"".",9,economia-credito-finanza
-it_67964,it,,,,"BRATISLAVA, 13 NOV - In un incidente stradale avvenuto nel sud della Slovacchia sono morti 12 studenti, di cui quattro minorenni. Altri 15 giovani sono rimasti feriti, tre in maniera grave. Secondo i media slovacchi, il conducente di un camion carico di pietre ha perso il controllo del veicolo, che si è scontrato con un autobus di studenti fra le città di Nitra e Zlate Moravce. Si tratta del più grave incidente stradale mai avvenuto in Slovacchia dal 1995, quando un autobus ceco sfondò una barriera di sicurezza nei pressi di Kolarovice (nordovest del Paese) precipitando per 25 metri e causando 17 morti.. YK1-CAL ",25,trasporti

examples/results/sample-dataset_240312_133454.csv

@@ -1,10 +0,0 @@
-doc_id,document_language,gfun_prediction,gfun_string_prediction
-it_74231,it,27,vita_e_cultura_religiosa
-it_39668,it,19,politica-partiti-istituzioni-sindacati
-it_74956,it,27,vita_e_cultura_religiosa
-it_46451,it,16,lavoro-previdenza
-it_14143,it,6,consumi-servizi
-it_60547,it,22,scienze-tecnologie
-it_24656,it,16,lavoro-previdenza
-it_27162,it,6,consumi-servizi
-it_67964,it,7,cronaca

gfun/csvlogger.py

@@ -1,32 +0,0 @@
-import csv
-import pandas as pd
-import os
-
-class CsvLogger:
-    def __init__(self, outfile="log.csv"):
-        self.outfile = outfile
-        # self.init_logfile()
-    
-    # def init_logfile(self):
-    #     if not os.path.isfile(self.outfile.replace(".csv", ".avg.csv")):
-    #         os.makedirs(self.outfile.replace(".csv", ".avg.csv"), exist_ok=True)
-    #     if not os.path.isfile(self.outfile.replace(".csv", ".lang.avg.csv")):
-    #         os.makedirs(self.outfile.replace(".csv", ".lang.csv"), exist_ok=True)
-    #     return
-    
-    def log_lang_results(self, results: dict, config="gfun-default", notes=None):
-        df = pd.DataFrame.from_dict(results, orient="columns")
-        df["config"] = config["gFun"]["simple_id"]
-        df["aggfunc"] = config["gFun"]["aggfunc"]
-        df["dataset"] = config["gFun"]["dataset"]
-        df["id"] = config["gFun"]["id"]
-        df["optimc"] = config["gFun"]["optimc"]
-        df["timing"] = config["gFun"]["timing"]
-        df["notes"] = notes
-        with open(self.outfile, 'a') as f:
-            df.to_csv(f, mode='a', header=f.tell()==0) 
-
-        
-
-
-

gfun/generalizedFunnelling.py

@@ -8,43 +8,47 @@ from gfun.vgfs.learners.svms import MetaClassifier, get_learner
 from gfun.vgfs.multilingualGen import MultilingualGen
 from gfun.vgfs.textualTransformerGen import TextualTransformerGen
 from gfun.vgfs.vanillaFun import VanillaFunGen
+from gfun.vgfs.visualTransformerGen import VisualTransformerGen
 from gfun.vgfs.wceGen import WceGen
 
 
 class GeneralizedFunnelling:
     def __init__(
         self,
+        posterior,
+        wce,
+        multilingual,
+        textual_transformer,
+        visual_transformer,
         langs,
         num_labels,
+        classification_type,
+        embed_dir,
+        n_jobs,
+        batch_size,
+        eval_batch_size,
+        max_length,
+        textual_lr,
+        visual_lr,
+        epochs,
+        patience,
+        evaluate_step,
+        textual_transformer_name,
+        visual_transformer_name,
+        optimc,
+        device,
+        load_trained,
         dataset_name,
-        posterior=True,
-        wce=False,
-        multilingual=False,
-        textual_transformer=False,
-        classification_type="multilabel",
-        embed_dir="embeddings/muse",
-        n_jobs=-1,
-        batch_size=32,
-        eval_batch_size=128,
-        max_length=512,
-        textual_lr=1e-4,
-        epochs=50,
-        patience=10,
-        evaluate_step=25,
-        optimc=True,
-        device="cuda:0",
-        load_trained=None,
-        probabilistic=True,
-        aggfunc="mean",
-        load_meta=False,
-        trained_text_trf=None,
-        textual_transformer_name="mbert",
+        probabilistic,
+        aggfunc,
+        load_meta,
     ):
         # Setting VFGs -----------
         self.posteriors_vgf = posterior
         self.wce_vgf = wce
         self.multilingual_vgf = multilingual
         self.textual_trf_vgf = textual_transformer
+        self.visual_trf_vgf = visual_transformer
         self.probabilistic = probabilistic
         self.num_labels = num_labels
         self.clf_type = classification_type
@@ -54,7 +58,6 @@ class GeneralizedFunnelling:
         self.cached = True
         # Textual Transformer VGF params ----------
         self.textual_trf_name = textual_transformer_name
-        self.trained_text_trf = trained_text_trf
         self.epochs = epochs
         self.textual_trf_lr = textual_lr
         self.textual_scheduler = "ReduceLROnPlateau"
@@ -65,6 +68,10 @@ class GeneralizedFunnelling:
         self.patience = patience
         self.evaluate_step = evaluate_step
         self.device = device
+        # Visual Transformer VGF params ----------
+        self.visual_trf_name = visual_transformer_name
+        self.visual_trf_lr = visual_lr
+        self.visual_scheduler = "ReduceLROnPlateau"
         # Metaclassifier params ------------
         self.optimc = optimc
         # -------------------
@@ -117,15 +124,6 @@ class GeneralizedFunnelling:
                     epochs=self.epochs,
                     attn_stacking_type=attn_stacking,
                 )
-
-            self._model_id = get_unique_id(
-                self.dataset_name,
-                self.posteriors_vgf,
-                self.multilingual_vgf,
-                self.wce_vgf,
-                self.textual_trf_vgf,
-                self.aggfunc,
-            )
             return self
 
         if self.posteriors_vgf:
@@ -166,10 +164,26 @@ class GeneralizedFunnelling:
                 patience=self.patience,
                 device=self.device,
                 classification_type=self.clf_type,
-                saved_model=self.trained_text_trf,
             )
             self.first_tier_learners.append(transformer_vgf)
 
+        if self.visual_trf_vgf:
+            visual_trasformer_vgf = VisualTransformerGen(
+                dataset_name=self.dataset_name,
+                model_name="vit",
+                lr=self.visual_trf_lr,
+                scheduler=self.visual_scheduler,
+                epochs=self.epochs,
+                batch_size=self.batch_size_trf,
+                batch_size_eval=self.eval_batch_size_trf,
+                probabilistic=self.probabilistic,
+                evaluate_step=self.evaluate_step,
+                patience=self.patience,
+                device=self.device,
+                classification_type=self.clf_type,
+            )
+            self.first_tier_learners.append(visual_trasformer_vgf)
+
         if "attn" in self.aggfunc:
             attn_stacking = self.aggfunc.split("_")[1]
             self.attn_aggregator = AttentionAggregator(
@@ -195,6 +209,7 @@ class GeneralizedFunnelling:
             self.multilingual_vgf,
             self.wce_vgf,
             self.textual_trf_vgf,
+            self.visual_trf_vgf,
             self.aggfunc,
         )
         print(f"- model id: {self._model_id}")
@@ -226,7 +241,7 @@ class GeneralizedFunnelling:
                 self.metaclassifier.fit(agg, lY)
             return self
 
-        self.vectorizer.fit(lX) # TODO this should fit also out-of-voc languages (for muses)
+        self.vectorizer.fit(lX)
         self.init_vgfs_vectorizers()
 
         projections = []
@@ -240,9 +255,8 @@ class GeneralizedFunnelling:
 
         return self
 
-    def transform(self, lX, output_ids=False):
+    def transform(self, lX):
         projections = []
-        l_ids = {}
         for vgf in self.first_tier_learners:
             l_posteriors = vgf.transform(lX)
             projections.append(l_posteriors)
@@ -251,10 +265,6 @@ class GeneralizedFunnelling:
         if self.clf_type == "singlelabel":
             for lang, preds in l_out.items():
                 l_out[lang] = predict(preds, clf_type=self.clf_type)
-                l_ids[lang] = lX[lang]["id"]
-        if output_ids:
-            return l_out, l_ids
-        else:
         return l_out
 
     def fit_transform(self, lX, lY):
@@ -289,47 +299,31 @@ class GeneralizedFunnelling:
         return aggregated
 
     def _aggregate_mean(self, first_tier_projections):
-        # aggregated = {
-        #     lang: np.zeros(data.shape)
-        #     for lang, data in first_tier_projections[0].items()
-        # }
-        aggregated = {}
+        aggregated = {
+            lang: np.zeros(data.shape)
+            for lang, data in first_tier_projections[0].items()
+        }
         for lang_projections in first_tier_projections:
             for lang, projection in lang_projections.items():
-                if lang not in aggregated:
-                    aggregated[lang] = np.zeros(projection.shape)
                 aggregated[lang] += projection
 
-        def get_denom(lang, projs):
-            den = 0
-            for proj in projs:
-                if lang in proj:
-                    den += 1
-            return den
-
-        for lang, _ in aggregated.items():
-            # aggregated[lang] /= len(first_tier_projections)
-            aggregated[lang] /= get_denom(lang, first_tier_projections)
-        
-        
+        for lang, projection in aggregated.items():
+            aggregated[lang] /= len(first_tier_projections)
 
         return aggregated
 
     def get_config(self):
         c = {}
-        simple_config = ""
 
         for vgf in self.first_tier_learners:
             vgf_config = vgf.get_config()
-            c.update({vgf_config["name"]: vgf_config})
-            simple_config += vgf_config["simple_id"]
+            c.update(vgf_config)
 
         gfun_config = {
             "id": self._model_id,
             "aggfunc": self.aggfunc,
             "optimc": self.optimc,
             "dataset": self.dataset_name,
-            "simple_id": "".join(sorted(simple_config))
         }
 
         c["gFun"] = gfun_config
@@ -349,10 +343,8 @@ class GeneralizedFunnelling:
             self.save_first_tier_learners(model_id=self._model_id)
 
         if save_meta:
-            _basedir = os.path.join("models", "metaclassifier")
-            os.makedirs(_basedir) 
             with open(
-                os.path.join(_basedir, f"meta_{self._model_id}.pkl"),
+                os.path.join("models", "metaclassifier", f"meta_{self._model_id}.pkl"),
                 "wb",
             ) as f:
                 pickle.dump(self.metaclassifier, f)
@@ -380,7 +372,6 @@ class GeneralizedFunnelling:
                 "rb",
             ) as vgf:
                 first_tier_learners.append(pickle.load(vgf))
-                print(f"- loaded trained VanillaFun VGF")
         if self.multilingual_vgf:
             with open(
                 os.path.join(
@@ -389,7 +380,6 @@ class GeneralizedFunnelling:
                 "rb",
             ) as vgf:
                 first_tier_learners.append(pickle.load(vgf))
-                print(f"- loaded trained Multilingual VGF")
         if self.wce_vgf:
             with open(
                 os.path.join(
@@ -398,26 +388,20 @@ class GeneralizedFunnelling:
                 "rb",
             ) as vgf:
                 first_tier_learners.append(pickle.load(vgf))
-                print(f"- loaded trained WCE VGF")
         if self.textual_trf_vgf:
             with open(
                 os.path.join(
-                    "models",
-                    "vgfs",
-                    "textual_transformer",
-                    f"textualTransformerGen_{model_id}.pkl",
+                    "models", "vgfs", "transformer", f"transformerGen_{model_id}.pkl"
                 ),
                 "rb",
             ) as vgf:
                 first_tier_learners.append(pickle.load(vgf))
-                print(f"- loaded trained Textual Transformer VGF")
 
         if load_meta:
             with open(
                 os.path.join("models", "metaclassifier", f"meta_{model_id}.pkl"), "rb"
             ) as f:
                 metaclassifier = pickle.load(f)
-                print(f"- loaded trained metaclassifier")
         else:
             metaclassifier = None
         return first_tier_learners, metaclassifier, vectorizer
@@ -465,6 +449,7 @@ def get_unique_id(
     multilingual,
     wce,
     textual_transformer,
+    visual_transformer,
     aggfunc,
 ):
     from datetime import datetime
@@ -475,5 +460,6 @@ def get_unique_id(
     model_id += "m" if multilingual else ""
     model_id += "w" if wce else ""
     model_id += "t" if textual_transformer else ""
+    model_id += "v" if visual_transformer else ""
     model_id += f"_{aggfunc}"
     return f"{model_id}_{now}"

gfun/hf_trainer.py

@@ -1,227 +0,0 @@
-from os.path import expanduser, join
-
-import torch
-from transformers import (
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
-    DataCollatorWithPadding,
-    TrainingArguments,
-)
-from gfun.vgfs.commons import Trainer
-from datasets import load_dataset, DatasetDict
-
-from transformers import Trainer
-from pprint import pprint
-
-import transformers
-import evaluate
-import pandas as pd
-
-transformers.logging.set_verbosity_error()
-
-RAI_D_COLUMNS = ["id", "provider", "date", "title", "text", "label"]
-MAX_LEN = 128 
-
-
-def init_callbacks(patience=-1, nosave=False):
-    callbacks = []
-    if patience != -1 and not nosave:
-        callbacks.append(transformers.EarlyStoppingCallback(early_stopping_patience=patience))
-    return callbacks
-
-
-def init_model(model_name, nlabels, saved_model=None):
-    if model_name == "mbert":
-        if saved_model is None:
-            hf_name = "bert-base-multilingual-cased"
-        else:
-            hf_name = saved_model
-    elif model_name == "xlm-roberta":
-        if saved_model is None:
-            hf_name = "xlm-roberta-base"
-        else:
-            hf_name = saved_model
-    else:
-        raise NotImplementedError
-    tokenizer = AutoTokenizer.from_pretrained(hf_name)
-    model = AutoModelForSequenceClassification.from_pretrained(hf_name, num_labels=nlabels)
-    return tokenizer, model
-
-
-def main(args):
-    saved_model = args.savedmodel
-    trainlang = args.trainlangs
-    datapath = args.datapath
-
-    tokenizer, model = init_model(args.model, args.nlabels, saved_model=saved_model)
-
-    data = load_dataset(
-        "csv",
-        data_files = {
-            "train": expanduser(join(datapath, "train.csv")),
-            "test": expanduser(join(datapath, "test.small.csv"))
-        }
-    )
-
-    def filter_dataset(dataset, lang):
-        indices = [i for i, l in enumerate(dataset["lang"]) if l == lang]
-        dataset = dataset.select(indices)
-        return dataset
-    
-    if trainlang is not None:
-        data["train"] = filter_dataset(data["train"], lang=trainlang)
-
-    def process_sample_rai(sample):
-        inputs = [f"{title}. {text}" for title, text in  zip(sample["title"], sample["text"])]
-        labels = sample["label"]
-        model_inputs = tokenizer(inputs, max_length=MAX_LEN, truncation=True)
-        model_inputs["labels"] = labels
-        return model_inputs
-    
-    data = data.map(
-        process_sample_rai,
-        batched=True,
-        num_proc=4,
-        load_from_cache_file=True,
-        remove_columns=RAI_D_COLUMNS,
-    )
-    train_val_splits = data["train"].train_test_split(test_size=0.2, seed=42)
-    data.set_format("torch")
-    data = DatasetDict(
-        {
-            "train": train_val_splits["train"],
-            "validation": train_val_splits["test"],
-            "test": data["test"],
-        }
-    )
-
-    data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
-    callbacks = init_callbacks(args.patience, args.nosave)
-
-    f1_metric = evaluate.load("f1")
-    accuracy_metric = evaluate.load("accuracy")
-    precision_metric = evaluate.load("precision")
-    recall_metric = evaluate.load("recall")
-
-    training_args = TrainingArguments(
-        output_dir=f"hf_models/{args.model}-fewshot-full" if trainlang is None else f"hf_models/{args.model}-zeroshot-full",
-        run_name="model-zeroshot" if trainlang is not None else "model-fewshot",
-        do_train=True,
-        evaluation_strategy="steps",
-        per_device_train_batch_size=args.batch,
-        per_device_eval_batch_size=args.batch,
-        gradient_accumulation_steps=args.gradacc,
-        eval_accumulation_steps=10,
-        learning_rate=args.lr,
-        weight_decay=0.1,
-        max_grad_norm=5.0,
-        num_train_epochs=args.epochs,
-        lr_scheduler_type=args.scheduler,
-        warmup_ratio=0.01,
-        logging_strategy="steps",
-        logging_first_step=True,
-        logging_steps=args.steplog,
-        seed=42,
-        fp16=args.fp16,
-        load_best_model_at_end=False if args.nosave else True,
-        save_strategy="no" if args.nosave else "steps",
-        save_total_limit=2,
-        eval_steps=args.stepeval,
-        disable_tqdm=False,
-        log_level="warning",
-        report_to=["wandb"] if args.wandb else "none",
-        optim="adamw_torch",
-        save_steps=args.stepeval
-    )
-
-
-    def compute_metrics(eval_preds):
-        preds = eval_preds.predictions.argmax(-1)
-        targets = eval_preds.label_ids
-        setting = "macro"
-        f1_score_macro = f1_metric.compute(
-            predictions=preds, references=targets, average="macro"
-        )
-        f1_score_micro = f1_metric.compute(
-            predictions=preds, references=targets, average="micro"
-        )
-        accuracy_score = accuracy_metric.compute(predictions=preds, references=targets)
-        precision_score = precision_metric.compute(
-            predictions=preds, references=targets, average=setting, zero_division=1
-        )
-        recall_score = recall_metric.compute(
-            predictions=preds, references=targets, average=setting, zero_division=1
-        )
-        results = {
-            "macro_f1score": f1_score_macro["f1"],
-            "micro_f1score": f1_score_micro["f1"],
-            "accuracy": accuracy_score["accuracy"],
-            "precision": precision_score["precision"],
-            "recall": recall_score["recall"],
-        }
-        results = {k: round(v, 4) for k, v in results.items()}
-        return results
-
-    if args.wandb:
-        import wandb
-        wandb.init(entity="andreapdr", project=f"gfun", 
-                   name="model-zeroshot-full" if trainlang is not None else "model-fewshot-full",
-                   config=vars(args))
-
-    trainer = Trainer(
-        model=model,
-        args=training_args,
-        train_dataset=data["train"],
-        eval_dataset=data["validation"],
-        compute_metrics=compute_metrics,
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        callbacks=callbacks,
-    )
-
-    if not args.onlytest:
-        print("- Training:")
-        trainer.train()
-
-    print("- Testing:")
-    test_results = trainer.evaluate(eval_dataset=data["test"], metric_key_prefix="test")
-    test_results = trainer.predict(test_dataset=data["test"], metric_key_prefix="test")
-    pprint(test_results.metrics)
-    save_preds(data["test"], test_results.predictions, trainlang)
-    exit()
-
-def save_preds(dataset, predictions, trainlang=None):
-    df = pd.DataFrame()
-    df["langs"] = dataset["lang"]
-    df["labels"] = dataset["labels"]
-    df["preds"] = predictions.argmax(axis=1)
-    if trainlang is not None:
-        df.to_csv(f"results/zeroshot.{trainlang}.model.csv", index=False)
-    else:
-        df.to_csv("results/fewshot.model.csv", index=False)
-    return
-
-
-
-if __name__ == "__main__":
-    from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
-    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
-    parser.add_argument("--model",              type=str,   metavar="", default="mbert")
-    parser.add_argument("--nlabels",            type=int,   metavar="", default=28)
-    parser.add_argument("--lr",                 type=float, metavar="", default=5e-5,       help="Set learning rate",)
-    parser.add_argument("--scheduler",          type=str,   metavar="", default="cosine",   help="Accepted: [\"cosine\", \"cosine-reset\", \"cosine-warmup\", \"cosine-warmup-reset\", \"constant\"]")
-    parser.add_argument("--batch",              type=int,   metavar="", default=8,          help="Set batch size")
-    parser.add_argument("--gradacc",            type=int,   metavar="", default=1,          help="Gradient accumulation steps")
-    parser.add_argument("--epochs",             type=int,   metavar="", default=10,         help="Set epochs")
-    parser.add_argument("--stepeval",           type=int,   metavar="", default=50,         help="Run evaluation every n steps")
-    parser.add_argument("--steplog",            type=int,   metavar="", default=50,         help="Log training every n steps")
-    parser.add_argument("--patience",           type=int,   metavar="", default=10,         help="EarlyStopper patience")
-    parser.add_argument("--fp16",               action="store_true",                        help="Use fp16 precision")
-    parser.add_argument("--wandb",              action="store_true",                        help="Log to wandb")
-    parser.add_argument("--nosave",             action="store_true",                        help="Avoid saving model")
-    parser.add_argument("--onlytest",           action="store_true",                        help="Simply test model on test set")
-    parser.add_argument("--trainlang",          default=None, type=str,                     help="set training language for zero-shot experiments" )
-    parser.add_argument("--datapath",           type=str,               default="data",     help="path to the csv dataset. Dir should contain both a train.csv and a test.csv file")
-    parser.add_argument("--savedmodel",         type=str,               default="hf_models/mbert-rai-fewshot-second/checkpoint-9000")
-    args = parser.parse_args()
-    main(args)

gfun/vgfs/commons.py

@@ -104,11 +104,6 @@ class TfidfVectorizerMultilingual:
     def __init__(self, **kwargs):
         self.kwargs = kwargs
 
-    def update_vectorizer(self, X, lang):
-        self.langs.append(lang)
-        self.vectorizer[lang] = TfidfVectorizer(**self.kwargs).fit(X["text"])
-        return self
-
     def fit(self, lX, ly=None):
         self.langs = sorted(lX.keys())
         self.vectorizer = {
@@ -117,13 +112,7 @@ class TfidfVectorizerMultilingual:
         return self
 
     def transform(self, lX):
-        in_langs = lX.keys()
-        for in_l in in_langs:
-            if in_l not in self.langs:
-                print(f"[NB: found unvectorized language! Updatding vectorizer for {in_l=}]")
-                self.update_vectorizer(X=lX[in_l], lang=in_l)
-        # return {l: self.vectorizer[l].transform(lX[l]["text"]) for l in self.langs} # TODO we can update the vectorizer with new languages here!
-        return {l: self.vectorizer[l].transform(lX[l]["text"]) for l in in_langs}
+        return {l: self.vectorizer[l].transform(lX[l]["text"]) for l in self.langs}
 
     def fit_transform(self, lX, ly=None):
         return self.fit(lX, ly).transform(lX)

gfun/vgfs/learners/svms.py

@@ -173,18 +173,9 @@ class NaivePolylingualClassifier:
         :return: a dictionary of probabilities that each document belongs to each class
         """
         assert self.model is not None, "predict called before fit"
-        if not set(lX.keys()).issubset(set(self.model.keys())):
-            langs = set(lX.keys()).intersection(set(self.model.keys()))
-            scores = Parallel(n_jobs=self.n_jobs, max_nbytes=None)(
-                delayed(self.model[lang].predict_proba)(lX[lang]) for lang in langs)
-            # res = {lang: None for lang in lX.keys()}
-            # for i, lang in enumerate(langs):
-            #     res[lang] = scores[i]
-            # return res
-            return {lang: scores[i] for i, lang in enumerate(langs)}
-        # assert set(lX.keys()).issubset(
-        #     set(self.model.keys())
-        # ), "unknown languages requested in decision function"
+        assert set(lX.keys()).issubset(
+            set(self.model.keys())
+        ), "unknown languages requested in decision function"
         langs = list(lX.keys())
         scores = Parallel(n_jobs=self.n_jobs, max_nbytes=None)(
             delayed(self.model[lang].predict_proba)(lX[lang]) for lang in langs

gfun/vgfs/multilingualGen.py

@@ -55,22 +55,13 @@ class MultilingualGen(ViewGen):
         return self
 
     def transform(self, lX):
-        _langs = lX.keys()
         lX = self.vectorizer.transform(lX)
-        if _langs != sorted(self.vectorizer.vectorizer.keys()):
-            """Loading word-embeddings for unseen languages at training time (zero-shot scenario),
-            excluding (exclude=old_langs) already loaded matrices."""
-            old_langs = self.langs
-            self.langs = sorted(self.vectorizer.vectorizer.keys())
-            new_load, _ = self._load_embeddings(embed_dir=self.embed_dir, cached=self.cached, exclude=old_langs)
-            for k, v in new_load.items():
-                self.multi_embeddings[k] = v
 
         XdotMulti = Parallel(n_jobs=self.n_jobs)(
             delayed(XdotM)(lX[lang], self.multi_embeddings[lang], sif=self.sif)
-            for lang in _langs 
+            for lang in self.langs
         )
-        lZ = {lang: XdotMulti[i] for i, lang in enumerate(_langs)}
+        lZ = {lang: XdotMulti[i] for i, lang in enumerate(self.langs)}
         lZ = _normalize(lZ, l2=True)
         if self.probabilistic and self.fitted:
             lZ = self.feature2posterior_projector.transform(lZ)
@@ -79,12 +70,10 @@ class MultilingualGen(ViewGen):
     def fit_transform(self, lX, lY):
         return self.fit(lX, lY).transform(lX)
 
-    def _load_embeddings(self, embed_dir, cached, exclude=None):
+    def _load_embeddings(self, embed_dir, cached):
         if "muse" in self.embed_dir.lower():
-            if exclude is not None:
-                langs = set(self.langs) - set(exclude)
             multi_embeddings = load_MUSEs(
-                langs=self.langs if exclude is None else langs,
+                langs=self.langs,
                 l_vocab=self.vectorizer.vocabulary(),
                 dir_path=embed_dir,
                 cached=cached,
@@ -100,7 +89,6 @@ class MultilingualGen(ViewGen):
             "cached": self.cached,
             "sif": self.sif,
             "probabilistic": self.probabilistic,
-            "simple_id": "m"
         }
 
     def save_vgf(self, model_id):
@@ -176,8 +164,6 @@ def extract(l_voc, l_embeddings):
     """
     l_extracted = {}
     for lang, words in l_voc.items():
-        if lang not in l_embeddings:
-            continue
         source_id, target_id = reindex(words, l_embeddings[lang].stoi)
         extraction = torch.zeros((len(words), l_embeddings[lang].vectors.shape[-1]))
         extraction[source_id] = l_embeddings[lang].vectors[target_id]

gfun/vgfs/textualTransformerGen.py

@@ -19,7 +19,6 @@ from dataManager.torchDataset import MultilingualDatasetTorch
 
 transformers.logging.set_verbosity_error()
 
-# TODO should pass also attention_mask to transformer model!
 
 class MT5ForSequenceClassification(nn.Module):
     def __init__(self, model_name, num_labels, output_hidden_states):
@@ -46,12 +45,11 @@ class MT5ForSequenceClassification(nn.Module):
 
     def save_pretrained(self, checkpoint_dir):
         torch.save(self.state_dict(), checkpoint_dir + ".pt")
-        return self
+        return
 
     def from_pretrained(self, checkpoint_dir):
         checkpoint_dir += ".pt"
-        self.load_state_dict(torch.load(checkpoint_dir))
-        return self
+        return self.load_state_dict(torch.load(checkpoint_dir))
 
 
 class TextualTransformerGen(ViewGen, TransformerGen):
@@ -73,7 +71,6 @@ class TextualTransformerGen(ViewGen, TransformerGen):
         patience=5,
         classification_type="multilabel",
         scheduler="ReduceLROnPlateau",
-        saved_model = None
     ):
         super().__init__(
             self._validate_model_name(model_name),
@@ -92,7 +89,6 @@ class TextualTransformerGen(ViewGen, TransformerGen):
             n_jobs=n_jobs,
             verbose=verbose,
         )
-        self.saved_model = saved_model
         self.clf_type = classification_type
         self.fitted = False
         print(
@@ -103,7 +99,7 @@ class TextualTransformerGen(ViewGen, TransformerGen):
         if "bert" == model_name:
             return "bert-base-uncased"
         elif "mbert" == model_name:
-            return "bert-base-multilingual-cased"
+            return "bert-base-multilingual-uncased"
         elif "xlm-roberta" == model_name:
             return "xlm-roberta-base"
         elif "mt5" == model_name:
@@ -111,16 +107,12 @@ class TextualTransformerGen(ViewGen, TransformerGen):
         else:
             raise NotImplementedError
 
-    def load_pretrained_model(self, model_name, num_labels, saved_model=None):
+    def load_pretrained_model(self, model_name, num_labels):
         if model_name == "google/mt5-small":
             return MT5ForSequenceClassification(
                 model_name, num_labels=num_labels, output_hidden_states=True
             )
         else:
-            if saved_model:
-                model_name = saved_model
-            else:
-                model_name = "google/bert-base-multilingual-cased"
             return AutoModelForSequenceClassification.from_pretrained(
                 model_name, num_labels=num_labels, output_hidden_states=True
             )
@@ -128,8 +120,8 @@ class TextualTransformerGen(ViewGen, TransformerGen):
     def load_tokenizer(self, model_name):
         return AutoTokenizer.from_pretrained(model_name)
 
-    def init_model(self, model_name, num_labels, saved_model):
-        return self.load_pretrained_model(model_name, num_labels, saved_model), self.load_tokenizer(
+    def init_model(self, model_name, num_labels):
+        return self.load_pretrained_model(model_name, num_labels), self.load_tokenizer(
             model_name
         )
 
@@ -149,14 +141,64 @@ class TextualTransformerGen(ViewGen, TransformerGen):
         _l = list(lX.keys())[0]
         self.num_labels = lY[_l].shape[-1]
         self.model, self.tokenizer = self.init_model(
-            self.model_name, num_labels=self.num_labels, saved_model=self.saved_model,
+            self.model_name, num_labels=self.num_labels
         )
 
-        self.model.to("cuda")
+        tr_lX, tr_lY, val_lX, val_lY = self.get_train_val_data(
+            lX, lY, split=0.2, seed=42, modality="text"
+        )
+
+        tra_dataloader = self.build_dataloader(
+            tr_lX,
+            tr_lY,
+            processor_fn=self._tokenize,
+            torchDataset=MultilingualDatasetTorch,
+            batch_size=self.batch_size,
+            split="train",
+            shuffle=True,
+        )
+
+        val_dataloader = self.build_dataloader(
+            val_lX,
+            val_lY,
+            processor_fn=self._tokenize,
+            torchDataset=MultilingualDatasetTorch,
+            batch_size=self.batch_size_eval,
+            split="val",
+            shuffle=False,
+        )
+
+        experiment_name = f"{self.model_name.replace('/', '-')}-{self.epochs}-{self.batch_size}-{self.dataset_name}"
+
+        trainer = Trainer(
+            model=self.model,
+            optimizer_name="adamW",
+            lr=self.lr,
+            device=self.device,
+            loss_fn=torch.nn.CrossEntropyLoss(),
+            print_steps=self.print_steps,
+            evaluate_step=self.evaluate_step,
+            patience=self.patience,
+            experiment_name=experiment_name,
+            checkpoint_path=os.path.join(
+                "models",
+                "vgfs",
+                "transformer",
+                self._format_model_name(self.model_name),
+            ),
+            vgf_name="textual_trf",
+            classification_type=self.clf_type,
+            n_jobs=self.n_jobs,
+            scheduler_name=self.scheduler,
+        )
+        trainer.train(
+            train_dataloader=tra_dataloader,
+            eval_dataloader=val_dataloader,
+            epochs=self.epochs,
+        )
 
         if self.probabilistic:
-            transformed = self.transform(lX)
-            self.feature2posterior_projector.fit(transformed, lY)
+            self.feature2posterior_projector.fit(self.transform(lX), lY)
 
         self.fitted = True
 
@@ -180,9 +222,9 @@ class TextualTransformerGen(ViewGen, TransformerGen):
 
         self.model.eval()
         with torch.no_grad():
-            # TODO should pass also attention_mask !
             for input_ids, lang in dataloader:
                 input_ids = input_ids.to(self.device)
+                # TODO: check this
                 if isinstance(self.model, MT5ForSequenceClassification):
                     batch_embeddings = self.model(input_ids).pooled.cpu().numpy()
                 else:
@@ -235,4 +277,4 @@ class TextualTransformerGen(ViewGen, TransformerGen):
 
     def get_config(self):
         c = super().get_config()
-        return {"name": "textual-transformer VGF", "textual_trf": c, "simple_id": "t"}
+        return {"textual_trf": c}

gfun/vgfs/vanillaFun.py

@@ -65,6 +65,3 @@ class VanillaFunGen(ViewGen):
         with open(_path, "wb") as f:
             pickle.dump(self, f)
         return self
-
-    def get_config(self):
-        return {"name": "Vanilla Funnelling VGF", "simple_id": "p"}

gfun/vgfs/visualTransformerGen.py

@@ -0,0 +1,189 @@
+from collections import defaultdict
+
+import numpy as np
+import torch
+import transformers
+from PIL import Image
+from transformers import AutoImageProcessor, AutoModelForImageClassification
+
+from gfun.vgfs.commons import Trainer
+from gfun.vgfs.transformerGen import TransformerGen
+from gfun.vgfs.viewGen import ViewGen
+from dataManager.torchDataset import MultimodalDatasetTorch
+
+transformers.logging.set_verbosity_error()
+
+
+class VisualTransformerGen(ViewGen, TransformerGen):
+    def __init__(
+        self,
+        model_name,
+        dataset_name,
+        lr=1e-5,
+        scheduler="ReduceLROnPlateau",
+        epochs=10,
+        batch_size=32,
+        batch_size_eval=128,
+        evaluate_step=10,
+        device="cpu",
+        probabilistic=False,
+        patience=5,
+        classification_type="multilabel",
+    ):
+        super().__init__(
+            model_name,
+            dataset_name,
+            epochs=epochs,
+            lr=lr,
+            scheduler=scheduler,
+            batch_size=batch_size,
+            batch_size_eval=batch_size_eval,
+            device=device,
+            evaluate_step=evaluate_step,
+            patience=patience,
+            probabilistic=probabilistic,
+        )
+        self.clf_type = classification_type
+        self.fitted = False
+        print(
+            f"- init Visual TransformerModel model_name: {self.model_name}, device: {self.device}]"
+        )
+
+    def _validate_model_name(self, model_name):
+        if "vit" == model_name:
+            return "google/vit-base-patch16-224-in21k"
+        else:
+            raise NotImplementedError
+
+    def init_model(self, model_name, num_labels):
+        model = AutoModelForImageClassification.from_pretrained(
+            model_name, num_labels=num_labels, output_hidden_states=True
+        )
+        image_processor = AutoImageProcessor.from_pretrained(model_name)
+        return model, image_processor
+
+    def process_all(self, X):
+        # TODO: should be moved as a collate_fn to avoid this overhead
+        processed = self.image_preprocessor(
+            [Image.open(img).convert("RGB") for img in X], return_tensors="pt"
+        )
+        return processed["pixel_values"]
+
+    def fit(self, lX, lY):
+        print("- fitting Visual Transformer View Generating Function")
+        _l = list(lX.keys())[0]
+        self.num_labels = lY[_l].shape[-1]
+        self.model, self.image_preprocessor = self.init_model(
+            self._validate_model_name(self.model_name), num_labels=self.num_labels
+        )
+
+        tr_lX, tr_lY, val_lX, val_lY = self.get_train_val_data(
+            lX, lY, split=0.2, seed=42, modality="image"
+        )
+
+        tra_dataloader = self.build_dataloader(
+            tr_lX,
+            tr_lY,
+            processor_fn=self.process_all,
+            torchDataset=MultimodalDatasetTorch,
+            batch_size=self.batch_size,
+            split="train",
+            shuffle=True,
+        )
+
+        val_dataloader = self.build_dataloader(
+            val_lX,
+            val_lY,
+            processor_fn=self.process_all,
+            torchDataset=MultimodalDatasetTorch,
+            batch_size=self.batch_size_eval,
+            split="val",
+            shuffle=False,
+        )
+
+        experiment_name = (
+            f"{self.model_name}-{self.epochs}-{self.batch_size}-{self.dataset_name}"
+        )
+
+        trainer = Trainer(
+            model=self.model,
+            optimizer_name="adamW",
+            device=self.device,
+            loss_fn=torch.nn.CrossEntropyLoss(),
+            lr=self.lr,
+            print_steps=self.print_steps,
+            evaluate_step=self.evaluate_step,
+            patience=self.patience,
+            experiment_name=experiment_name,
+            checkpoint_path="models/vgfs/transformer",
+            vgf_name="visual_trf",
+            classification_type=self.clf_type,
+            n_jobs=self.n_jobs,
+        )
+
+        trainer.train(
+            train_dataloader=tra_dataloader,
+            eval_dataloader=val_dataloader,
+            epochs=self.epochs,
+        )
+
+        if self.probabilistic:
+            self.feature2posterior_projector.fit(self.transform(lX), lY)
+
+        self.fitted = True
+
+        return self
+
+    def transform(self, lX):
+        # forcing to only image modality
+        lX = {lang: data["image"] for lang, data in lX.items()}
+        _embeds = []
+        l_embeds = defaultdict(list)
+
+        dataloader = self.build_dataloader(
+            lX,
+            lY=None,
+            processor_fn=self.process_all,
+            torchDataset=MultimodalDatasetTorch,
+            batch_size=self.batch_size_eval,
+            split="whole",
+            shuffle=False,
+        )
+
+        self.model.eval()
+        with torch.no_grad():
+            for input_ids, lang in dataloader:
+                input_ids = input_ids.to(self.device)
+                out = self.model(input_ids).hidden_states[-1]
+                batch_embeddings = out[:, 0, :].cpu().numpy()
+                _embeds.append((batch_embeddings, lang))
+
+        for embed, lang in _embeds:
+            for sample_embed, sample_lang in zip(embed, lang):
+                l_embeds[sample_lang].append(sample_embed)
+
+        if self.probabilistic and self.fitted:
+            l_embeds = self.feature2posterior_projector.transform(l_embeds)
+        elif not self.probabilistic and self.fitted:
+            l_embeds = {lang: np.array(preds) for lang, preds in l_embeds.items()}
+
+        return l_embeds
+
+    def fit_transform(self, lX, lY):
+        return self.fit(lX, lY).transform(lX)
+
+    def save_vgf(self, model_id):
+        import pickle
+        from os import makedirs
+        from os.path import join
+
+        vgf_name = "visualTransformerGen"
+        _basedir = join("models", "vgfs", "visual_transformer")
+        makedirs(_basedir, exist_ok=True)
+        _path = join(_basedir, f"{vgf_name}_{model_id}.pkl")
+        with open(_path, "wb") as f:
+            pickle.dump(self, f)
+        return self
+
+    def get_config(self):
+        return {"visual_trf": super().get_config()}

gfun/vgfs/wceGen.py

@@ -38,7 +38,6 @@ class WceGen(ViewGen):
             "name": "Word-Class Embeddings VGF",
             "n_jobs": self.n_jobs,
             "sif": self.sif,
-            "simple_id": "w"
         }
 
     def save_vgf(self, model_id):

infer.py

@@ -1,113 +0,0 @@
-import os
-import pandas as pd
-from pathlib import Path
-from datetime import datetime
-
-from dataManager.gFunDataset import SimpleGfunDataset
-from gfun.generalizedFunnelling import GeneralizedFunnelling
-
-
-def main(args):
-    dataset = SimpleGfunDataset(
-        dataset_name=Path(args.datapath).stem,
-        datapath=args.datapath,
-        multilabel=False,
-        only_inference=True,
-        labels=args.nlabels,
-    )
-
-    lX, _ = dataset.test()
-
-    gfun = GeneralizedFunnelling(
-        dataset_name=dataset.get_name(),
-        langs=dataset.langs(),
-        num_labels=dataset.num_labels(),
-        classification_type="singlelabel",
-        embed_dir=args.muse_dir,
-        posterior=True,
-        multilingual=True,
-        textual_transformer=True,
-        load_trained=args.trained_gfun,
-        load_meta=True,
-    )
-
-    predictions, ids = gfun.transform(lX, output_ids=True)
-    save_inference_preds(
-        preds=predictions,
-        doc_ids=ids,
-        dataset_name=dataset.get_name(),
-        targets=None,
-        category_mapper=args.category_map,
-        outdir=args.outdir,
-    )
-
-def save_inference_preds(preds, dataset_name, doc_ids, targets=None, category_mapper=None, outdir="results/inference-preds"):
-    """
-    Parameters
-    ----------
-    preds : Dict[str: np.array]
-            Predictions produced by generalized-funnelling.
-    dataset_name: str
-                  Dataset name used as output file name. File is stored in directory defined by `output_dir`
-                  argument e.g. "<output_dir>/<dataset_name>.csv"
-    doc_ids: Dict[str: List[str]]
-             Dictionary storing list of document ids (as defined in the csv input file)
-    targets: Dict[str: np.array]
-             If availabel, target true labels will be written to output file to ease performance evaluation.
-             (default = None)
-    category_mapper: Path
-                     Path to the 'category_mapper' csv file storing the category names (str) for each target class (integer).
-                     If not None, gFun predictions will be converetd and stored as target string classes.
-                     (default=None) 
-    output_dir: Path
-                Dir where to store output csv file.
-                (default = results/inference-preds)
-    """
-    os.makedirs(outdir, exist_ok=True)
-    df = pd.DataFrame()
-    langs = sorted(preds.keys())
-    _ids = []
-    _preds = []
-    _targets = []
-    _langs = []
-    for lang in langs:
-        _preds.extend(preds[lang].argmax(axis=1).tolist())
-        _langs.extend([lang for i in range(len(preds[lang]))])
-        _ids.extend(doc_ids[lang])
-    
-    df["doc_id"] = _ids
-    df["document_language"] = _langs
-    df["gfun_prediction"] = _preds
-    
-    if targets is not None:
-        for lang in langs:
-            _targets.extend(targets[lang].argmax(axis=1).tolist())
-            df["document_true_label"] = _targets
-
-    if category_mapper is not None:
-        mapper = pd.read_csv(category_mapper).to_dict()["category"]
-        df["gfun_string_prediction"] = [mapper[p] for p in _preds]
-
-    timestamp = datetime.now()
-    formatted_timestamp = timestamp.strftime("%y%m%d_%H%M%S")
-
-    output_file = f"{outdir}/{dataset_name}_{formatted_timestamp}.csv"
-    print(f"Storing predicitons in: {output_file}")
-    df.to_csv(output_file, index=False)
-
-    return
-        
-
-if __name__ == "__main__":
-    from argparse import ArgumentParser
-    parser = ArgumentParser()
-    parser.add_argument("--datapath", required=True, type=str, help="path to csv file containing the documents to be classified")
-    parser.add_argument("--outdir", type=str, default="results/inference-preds", help="path to store csv file containing gfun predictions")
-    parser.add_argument("--category_map", type=str, default=None, help="path to csv file containing the mapping from label name to label id [str: id]")
-    parser.add_argument("--nlabels", type=int, default=28)
-    parser.add_argument("--muse_dir", type=str, default="embeddings", help="path to muse embeddings")
-    parser.add_argument("--trained_gfun", type=str, default="rai_pmt_mean_231029", help="name of the trained gfun instance")
-
-    args = parser.parse_args()
-    main(args)
-

main.py

@@ -1,16 +1,27 @@
 import os
+import wandb
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
 from argparse import ArgumentParser
 from time import time
 
-from csvlogger import CsvLogger
 from dataManager.utils import get_dataset
 from evaluation.evaluate import evaluate, log_eval
 from gfun.generalizedFunnelling import GeneralizedFunnelling
 
-import pandas as pd
-
 """
 TODO:
+    - Transformers VGFs:
+        - scheduler with warmup and cosine
+        - freeze params method
+    - General:
+        [!] zero-shot setup
+        - CLS dataset is loading only "books" domain data
+        - documents should be trimmed to the same length (for SVMs we are using way too long tokens)
+    - Attention Aggregator:
+        - experiment with weight init of Attention-aggregator
+        - FFNN posterior-probabilities' dependent
     - Docs:
         - add documentations sphinx
 """
@@ -26,12 +37,14 @@ def get_config_name(args):
         config_name += "M+"
     if args.textual_transformer:
         config_name += f"TT_{args.textual_trf_name}+"
+    if args.visual_transformer:
+        config_name += f"VT_{args.visual_trf_name}+"
     return config_name.rstrip("+")
 
 
 def main(args):
-    dataset = get_dataset(args.datadir, args)
-    lX, lY = dataset.training(merge_validation=True)
+    dataset = get_dataset(args.dataset, args)
+    lX, lY = dataset.training()
     lX_te, lY_te = dataset.test()
 
     tinit = time()
@@ -44,12 +57,13 @@ def main(args):
                 args.multilingual,
                 args.multilingual,
                 args.textual_transformer,
+                args.visual_transformer,
             ]
         ), "At least one of VGF must be True"
 
     gfun = GeneralizedFunnelling(
         # dataset params ----------------------
-        dataset_name=dataset.name,
+        dataset_name=args.dataset,
         langs=dataset.langs(),
         num_labels=dataset.num_labels(),
         classification_type=args.clf_type,
@@ -63,16 +77,24 @@ def main(args):
         # Transformer VGF params --------------
         textual_transformer=args.textual_transformer,
         textual_transformer_name=args.textual_trf_name,
-        # trained_text_trf="hf_models/mbert-zeroshot-rai/checkpoint-1350",
-        trained_text_trf="hf_models/mbert-fewshot-rai-full/checkpoint-5150",
         batch_size=args.batch_size,
         eval_batch_size=args.eval_batch_size,
         epochs=args.epochs,
         textual_lr=args.textual_lr,
+        visual_lr=args.visual_lr,
         max_length=args.max_length,
         patience=args.patience,
         evaluate_step=args.evaluate_step,
         device=args.device,
+        # Visual Transformer VGF params --------------
+        visual_transformer=args.visual_transformer,
+        visual_transformer_name=args.visual_trf_name,
+        # batch_size=args.batch_size,
+        # epochs=args.epochs,
+        # lr=args.lr,
+        # patience=args.patience,
+        # evaluate_step=args.evaluate_step,
+        # device="cuda",
         # General params ---------------------
         probabilistic=args.features,
         aggfunc=args.aggfunc,
@@ -84,14 +106,11 @@ def main(args):
 
     config = gfun.get_config()
 
-    if args.wandb:
-        import wandb
     wandb.init(project="gfun", name=f"gFun-{get_config_name(args)}", config=config)
 
     gfun.fit(lX, lY)
 
-    # if args.load_trained is None and not args.nosave:
-    print("saving model")
+    if args.load_trained is None and not args.nosave:
         gfun.save(save_first_tier=True, save_meta=True)
 
     timetr = time()
@@ -130,33 +149,8 @@ def main(args):
             )
             wandb.log(gfun_res)
 
-    if args.wandb:
     log_barplot_wandb(lang_metrics_gfun, title_affix="per language")
-
-    config["gFun"]["timing"] = f"{timeval - tinit:.2f}"
-    csvlogger = CsvLogger(outfile="results/gfun.log.csv").log_lang_results(lang_metrics_gfun, config, notes="")
-    save_preds(gfun_preds, lY_te, config=config["gFun"]["simple_id"], dataset=config["gFun"]["dataset"])
-
-
-def save_preds(preds, targets, config="unk", dataset="unk"):
-    os.makedirs("results/preds")
-    df = pd.DataFrame()
-    langs = sorted(preds.keys())
-    _preds = []
-    _targets = []
-    _langs = []
-    for lang in langs:
-        _preds.extend(preds[lang].argmax(axis=1).tolist())
-        if targets is None:
-            _targets.extend(["na" for i in range(len(preds[lang]))])
-        else:
-            _targets.extend(targets[lang].argmax(axis=1).tolist())
-        _langs.extend([lang for i in range(len(preds[lang]))])
-    df["langs"] = _langs
-    df["labels"] = _targets 
-    df["preds"] = _preds
-    print(f"- storing predictions in 'results/preds/preds.gfun.{config}.{dataset}.csv'")
-    df.to_csv(f"results/preds/preds.gfun.{config}.{dataset}.csv", index=False)
+    log_barplot_wandb(avg_metrics_gfun, title_affix="averages")
 
 
 if __name__ == "__main__":
@@ -165,10 +159,8 @@ if __name__ == "__main__":
     parser.add_argument("--meta", action="store_true")
     parser.add_argument("--nosave", action="store_true")
     parser.add_argument("--device", type=str, default="cuda")
-    parser.add_argument("--tr_langs", nargs="+", default=None)
-    parser.add_argument("--te_langs", nargs="+", default=None)
     # Dataset parameters -------------------
-    parser.add_argument("-d", "--datadir", type=str, default=None, help="dir to dataset. It should contain both a train.csv and a test.csv file")
+    parser.add_argument("-d", "--dataset", type=str, default="rcv1-2")
     parser.add_argument("--domains", type=str, default="all")
     parser.add_argument("--nrows", type=int, default=None)
     parser.add_argument("--min_count", type=int, default=10)
@@ -180,12 +172,13 @@ if __name__ == "__main__":
     parser.add_argument("-m", "--multilingual", action="store_true")
     parser.add_argument("-w", "--wce", action="store_true")
     parser.add_argument("-t", "--textual_transformer", action="store_true")
+    parser.add_argument("-v", "--visual_transformer", action="store_true")
     parser.add_argument("--n_jobs", type=int, default=-1)
     parser.add_argument("--optimc", action="store_true")
     parser.add_argument("--features", action="store_false")
     parser.add_argument("--aggfunc", type=str, default="mean")
     # transformer parameters ---------------
-    parser.add_argument("--epochs", type=int, default=5)
+    parser.add_argument("--epochs", type=int, default=100)
     parser.add_argument("--textual_trf_name", type=str, default="mbert")
     parser.add_argument("--batch_size", type=int, default=32)
     parser.add_argument("--eval_batch_size", type=int, default=128)
@@ -193,9 +186,9 @@ if __name__ == "__main__":
     parser.add_argument("--max_length", type=int, default=128)
     parser.add_argument("--patience", type=int, default=5)
     parser.add_argument("--evaluate_step", type=int, default=10)
-    parser.add_argument("--reduced", action="store_true", help="run on reduced set of documents")
-    # logging
-    parser.add_argument("--wandb", action="store_true")
+    # Visual Transformer parameters --------------
+    parser.add_argument("--visual_trf_name", type=str, default="vit")
+    parser.add_argument("--visual_lr", type=float, default=1e-4)
 
     args = parser.parse_args()
 

plotters/distributions.py

@@ -0,0 +1,60 @@
+import matplotlib.pyplot as plt
+import datetime
+
+
+def plot_distribution(
+    x,
+    y,
+    labels,
+    title,
+    figsize=(10, 5),
+    logscale=False,
+    notes="",
+    max_labels=-1,
+    save=False,
+    path=None,
+):
+    # sort values and labels accordingly
+    y, labels = zip(*sorted(zip(y, labels), reverse=True))
+
+    if max_labels != -1:
+        x = x[:max_labels]
+        y = y[:max_labels]
+        labels = labels[:max_labels]
+
+    plt.figure(figsize=figsize)
+    plt.bar(x, y)
+    plt.xticks(x, labels, rotation=90)
+
+    if len(notes) != 0:
+        _title = f"{title} - {notes}"
+        if max_labels != -1:
+            _title += f" - Showing {max_labels} top labels"
+
+    plt.title(_title)
+
+    if logscale:
+        plt.yscale("symlog")
+
+    plt.tight_layout()
+
+    # plt.show()
+    if save:
+        now = datetime.datetime.now()
+        path = f"{path}/{title}_{now.strftime('%m%d_%H%M')}.png"
+        plt.savefig(path)
+    plt.close()
+
+
+def plot_histogram(x, title, figsize=(10, 5), save=False, path=None):
+    plt.figure(figsize=figsize)
+    plt.hist(x)
+    # plt.xticks(x, lables, rotation=90)
+    plt.yscale("symlog")
+    plt.title(title)
+    # plt.show()
+    if save:
+        now = datetime.datetime.now()
+        path = f"{path}/{title}_{now.strftime('%m%d_%H%M')}.png"
+        plt.savefig(path)
+    plt.close()

readme.md

@@ -1,36 +0,0 @@
-# gFun - RAI
-
-## Setup:
-```
-git clone https://gitea-s2i2s.isti.cnr.it/andrea.pedrotti/gfun_multimodal.git
-cd gfun_multimodal
-mkdir models
-mkdir resources
-# optional
-mkdir models/category_mappers
-```
-In `models`, scaricare i modelli pre-trained condivisi. La directory `models` contiene 3 subdir `metaclassifier, vgfs, vectorizer`.
-In `resources` estrarre i muse-embeddings.
-In `models/category_mappers` estrarre il file csv che contiene il mapping da category label a category id (opzionale).
-
-## Inference:
-Per eseguire la classificazione dei documenti:
-
-```python
-python infer.py --datapth <path/to/the/csv_file.csv>
-```
-
-I risultati saranno salvati di default nella cartella `results/inference-preds`, in un file csv denominato a seconda input file specificato in `--datapath` + il timetamp della run (e.g., `<csv_file>_<240312_13345>.csv`) (è possibile cambiare directory di output tramite `--outdir <my/output/dir/>`)
-
-NB: per ottenere i nomi (stringhe) delle classi predette è necessario specificare il path del file csv che contiene il mapping class id -> class label (argomento `--category_map`).
-
-```
-optional arguments:
-  -h, --help            show this help message and exit
-  --datapath            path to csv file containing the documents to be classified
-  --outdir              path to store csv file containing gfun predictions (default=results/inference-preds)
-  --category_map         path to csv file containing the mapping from label name to label id [str: id] (default=None)
-  --nlabels             number of target classes defined in the annotation schema (default=28)
-  --muse_dir            path to muse embeddings
-  --trained_gfun        name of the trained gfun instance
-```

requirements.txt

@@ -1,14 +1,13 @@
-beautifulsoup4==4.12.2
-datasets==2.13.0
+beautifulsoup4==4.11.2
 joblib==1.2.0
 matplotlib==3.6.3
 numpy==1.24.1
 pandas==1.5.3
-Pillow==10.0.1
+Pillow==9.4.0
 requests==2.28.2
-scikit_learn==1.3.1
-scipy==1.11.3
+scikit_learn==1.2.2
+scipy==1.10.1
 torch==1.13.1
 torchtext==0.14.1
 tqdm==4.64.1
-transformers==4.30.0
+transformers==4.26.0