From 2a5d0243db7b7e5c89ae1fdca1944d7e4b3a2455 Mon Sep 17 00:00:00 2001
From: andrea <andrea.pdr@hotmail.it>
Date: Thu, 28 Nov 2019 18:39:19 +0100
Subject: [PATCH] first commit

---
 .idea/deployment.xml                          |  20 +
 .idea/encodings.xml                           |   4 +
 .idea/misc.xml                                |  10 +
 .idea/modules.xml                             |   8 +
 .idea/tesi_funneling.iml                      |  14 +
 .idea/vcs.xml                                 |   6 +
 .idea/webServers.xml                          |  15 +
 .idea/workspace.xml                           | 655 ++++++++++++++++++
 src/.gitignore                                |   1 +
 src/FPEC_andrea.py                            | 151 ++++
 .../dataset_builder.cpython-37.pyc            | Bin 0 -> 24002 bytes
 src/data/__init__.py                          |   0
 src/data/__pycache__/__init__.cpython-37.pyc  | Bin 0 -> 144 bytes
 .../__pycache__/embeddings.cpython-37.pyc     | Bin 0 -> 9387 bytes
 src/data/__pycache__/languages.cpython-37.pyc | Bin 0 -> 1048 bytes
 .../__pycache__/supervised.cpython-37.pyc     | Bin 0 -> 2286 bytes
 .../text_preprocessor.cpython-37.pyc          | Bin 0 -> 1698 bytes
 .../__pycache__/tsr_function__.cpython-37.pyc | Bin 0 -> 9820 bytes
 src/data/embeddings.py                        | 196 ++++++
 src/data/languages.py                         |  42 ++
 src/data/reader/__init__.py                   |   0
 .../__pycache__/__init__.cpython-37.pyc       | Bin 0 -> 151 bytes
 .../jrcacquis_reader.cpython-37.pyc           | Bin 0 -> 10949 bytes
 .../__pycache__/rcv_reader.cpython-37.pyc     | Bin 0 -> 6717 bytes
 .../wikipedia_tools.cpython-37.pyc            | Bin 0 -> 10598 bytes
 src/data/reader/jrcacquis_reader.py           | 321 +++++++++
 src/data/reader/rcv_reader.py                 | 225 ++++++
 src/data/reader/wikipedia_tools.py            | 304 ++++++++
 src/data/supervised.py                        |  75 ++
 src/data/text_preprocessor.py                 |  33 +
 src/data/tsr_function__.py                    | 270 ++++++++
 src/dataset_builder.py                        | 567 +++++++++++++++
 .../__pycache__/learners.cpython-37.pyc       | Bin 0 -> 27631 bytes
 src/learning/learners.py                      | 646 +++++++++++++++++
 src/results/results.csv                       |   7 +
 src/transformers/__init__.py                  |   0
 src/transformers/clesa.py                     | 110 +++
 src/transformers/dci.py                       | 154 ++++
 src/transformers/riboc.py                     |  53 ++
 .../__pycache__/evaluation.cpython-37.pyc     | Bin 0 -> 4195 bytes
 src/util/__pycache__/file.cpython-37.pyc      | Bin 0 -> 1718 bytes
 src/util/__pycache__/metrics.cpython-37.pyc   | Bin 0 -> 6430 bytes
 src/util/__pycache__/results.cpython-37.pyc   | Bin 0 -> 2041 bytes
 src/util/evaluation.py                        |  95 +++
 src/util/file.py                              |  36 +
 src/util/metrics.py                           | 168 +++++
 src/util/results.py                           |  33 +
 47 files changed, 4219 insertions(+)
 create mode 100644 .idea/deployment.xml
 create mode 100644 .idea/encodings.xml
 create mode 100644 .idea/misc.xml
 create mode 100644 .idea/modules.xml
 create mode 100644 .idea/tesi_funneling.iml
 create mode 100644 .idea/vcs.xml
 create mode 100644 .idea/webServers.xml
 create mode 100644 .idea/workspace.xml
 create mode 100644 src/.gitignore
 create mode 100644 src/FPEC_andrea.py
 create mode 100644 src/__pycache__/dataset_builder.cpython-37.pyc
 create mode 100644 src/data/__init__.py
 create mode 100644 src/data/__pycache__/__init__.cpython-37.pyc
 create mode 100644 src/data/__pycache__/embeddings.cpython-37.pyc
 create mode 100644 src/data/__pycache__/languages.cpython-37.pyc
 create mode 100644 src/data/__pycache__/supervised.cpython-37.pyc
 create mode 100644 src/data/__pycache__/text_preprocessor.cpython-37.pyc
 create mode 100644 src/data/__pycache__/tsr_function__.cpython-37.pyc
 create mode 100644 src/data/embeddings.py
 create mode 100644 src/data/languages.py
 create mode 100644 src/data/reader/__init__.py
 create mode 100644 src/data/reader/__pycache__/__init__.cpython-37.pyc
 create mode 100644 src/data/reader/__pycache__/jrcacquis_reader.cpython-37.pyc
 create mode 100644 src/data/reader/__pycache__/rcv_reader.cpython-37.pyc
 create mode 100644 src/data/reader/__pycache__/wikipedia_tools.cpython-37.pyc
 create mode 100644 src/data/reader/jrcacquis_reader.py
 create mode 100644 src/data/reader/rcv_reader.py
 create mode 100644 src/data/reader/wikipedia_tools.py
 create mode 100755 src/data/supervised.py
 create mode 100644 src/data/text_preprocessor.py
 create mode 100755 src/data/tsr_function__.py
 create mode 100644 src/dataset_builder.py
 create mode 100644 src/learning/__pycache__/learners.cpython-37.pyc
 create mode 100644 src/learning/learners.py
 create mode 100644 src/results/results.csv
 create mode 100644 src/transformers/__init__.py
 create mode 100644 src/transformers/clesa.py
 create mode 100644 src/transformers/dci.py
 create mode 100644 src/transformers/riboc.py
 create mode 100644 src/util/__pycache__/evaluation.cpython-37.pyc
 create mode 100644 src/util/__pycache__/file.cpython-37.pyc
 create mode 100644 src/util/__pycache__/metrics.cpython-37.pyc
 create mode 100644 src/util/__pycache__/results.cpython-37.pyc
 create mode 100644 src/util/evaluation.py
 create mode 100644 src/util/file.py
 create mode 100644 src/util/metrics.py
 create mode 100644 src/util/results.py

diff --git a/.idea/deployment.xml b/.idea/deployment.xml
new file mode 100644
index 0000000..d6ba90a
--- /dev/null
+++ b/.idea/deployment.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" serverName="anna_isti">
+    <serverData>
+      <paths name="anna_isti">
+        <serverdata>
+          <mappings>
+            <mapping deploy="/home/andreapdr/funneling_pdr" local="$PROJECT_DIR$" web="/" />
+            <mapping deploy="/home/andreapdr/CLESA/embeddings" local="/storage/andrea/FUNNELING/embeddings" />
+          </mappings>
+          <excludedPaths>
+            <excludedPath local="true" path="$PROJECT_DIR$/src/venv" />
+            <excludedPath local="true" path="$PROJECT_DIR$/src/pickles/jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.pickle" />
+            <excludedPath local="true" path="$PROJECT_DIR$/src/results/results.csv" />
+          </excludedPaths>
+        </serverdata>
+      </paths>
+    </serverData>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
new file mode 100644
index 0000000..15a15b2
--- /dev/null
+++ b/.idea/encodings.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with NO BOM" />
+</project>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..9fa0db4
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="NodePackageJsonFileManager">
+    <packageJsonPaths />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (word-class-embeddings)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..5ddd5cc
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/tesi_funneling.iml" filepath="$PROJECT_DIR$/.idea/tesi_funneling.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/tesi_funneling.iml b/.idea/tesi_funneling.iml
new file mode 100644
index 0000000..828b8c4
--- /dev/null
+++ b/.idea/tesi_funneling.iml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/src/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.7 (word-class-embeddings)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/webServers.xml b/.idea/webServers.xml
new file mode 100644
index 0000000..56d5a5b
--- /dev/null
+++ b/.idea/webServers.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="WebServers">
+    <option name="servers">
+      <webServer id="8f0f329c-a17c-48ba-b459-18d8b1a104e5" name="anna_isti" url="http://anna.isti.cnr.it">
+        <fileTransfer host="anna.isti.cnr.it" port="22" accessType="SFTP">
+          <advancedOptions>
+            <advancedOptions dataProtectionLevel="Private" passiveMode="true" shareSSLContext="true" />
+          </advancedOptions>
+          <option name="port" value="22" />
+        </fileTransfer>
+      </webServer>
+    </option>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 0000000..92b2f97
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,655 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="f02472ae-5920-4b7a-abc6-0eb6f03dd03f" name="Default Changelist" comment="" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="CoverageDataManager">
+    <SUITE FILE_PATH="coverage/tesi_funneling$funneling_poly.coverage" NAME="funneling_poly Coverage Results" MODIFIED="1574690332154" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/src" />
+    <SUITE FILE_PATH="coverage/tesi_funneling$last_test.coverage" NAME="last_test Coverage Results" MODIFIED="1574960066673" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/src" />
+    <SUITE FILE_PATH="coverage/tesi_funneling$scratch.coverage" NAME="scratch Coverage Results" MODIFIED="1574759452703" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/src" />
+  </component>
+  <component name="FileEditorManager">
+    <splitter split-orientation="horizontal" split-proportion="0.5">
+      <split-first>
+        <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+          <file pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/torchtext/vocab.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="182">
+                  <caret line="321" selection-start-line="321" selection-end-line="321" />
+                  <folding>
+                    <element signature="e#13891#17737#0" />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+        </leaf>
+      </split-first>
+      <split-second>
+        <leaf>
+          <file pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/src/data/embeddings.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="462">
+                  <caret line="162" selection-start-line="162" selection-end-line="162" />
+                  <folding>
+                    <element signature="e#0#9#0" expanded="true" />
+                    <element signature="e#222#778#0" />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+        </leaf>
+      </split-second>
+    </splitter>
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>NaivePolylingualClassifier</find>
+      <find>tra</find>
+      <find>base_l</find>
+      <find>tfidf</find>
+      <find>proba</find>
+      <find>we</find>
+      <find>Wordembeddings</find>
+      <find>hstack</find>
+      <find>ha</find>
+      <find>timeit</find>
+      <find>ti</find>
+      <find>time</find>
+      <find>dot</find>
+      <find>vec</find>
+      <find>_fit_binary</find>
+      <find>oneVs</find>
+      <find>embed</find>
+      <find>no tf-</find>
+      <find>embedding_matrix</find>
+      <find>WordEm</find>
+      <find>WordEmbeddings</find>
+      <find># pretrai</find>
+      <find># [pre</find>
+      <find>joblib</find>
+    </findStrings>
+  </component>
+  <component name="HighlightingSettingsPerFile">
+    <setting file="file://$PROJECT_DIR$/src/learning/learners.py" root0="FORCE_HIGHLIGHTING" />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/src/dataset_builder.py" />
+        <option value="$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/multiclass.py" />
+        <option value="$PROJECT_DIR$/src/data/embeddings_fasttext.py" />
+        <option value="$PROJECT_DIR$/src/scratch.py" />
+        <option value="$PROJECT_DIR$/src/data/tsr_function__.py" />
+        <option value="$PROJECT_DIR$/src/data/supervised.py" />
+        <option value="$PROJECT_DIR$/src/last_test.py" />
+        <option value="&lt;8f0f329c-a17c-48ba-b459-18d8b1a104e5&gt;/home/andreapdr/funneling_pdr/src/data/embeddings.py" />
+        <option value="$PROJECT_DIR$/src/util/results.py" />
+        <option value="$PROJECT_DIR$/src/transformers/clesa.py" />
+        <option value="$PROJECT_DIR$/src/transformers/dci.py" />
+        <option value="$PROJECT_DIR$/src/util/evaluation.py" />
+        <option value="$PROJECT_DIR$/src/util/disable_sklearn_warnings.py" />
+        <option value="$PROJECT_DIR$/src/funneling_poly.py" />
+        <option value="$PROJECT_DIR$/src/learning/learners.py" />
+        <option value="$PROJECT_DIR$/src/FPEC_andrea.py" />
+        <option value="$PROJECT_DIR$/src/data/embeddings.py" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectFrameBounds" extendedState="6">
+    <option name="x" value="5" />
+    <option name="y" value="28" />
+    <option name="width" value="960" />
+    <option name="height" value="1052" />
+  </component>
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="Scope" />
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+              <item name="learning" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+              <item name="pickles" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+              <item name="results" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+              <item name="transformers" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="tesi_funneling" type="b2602c69:ProjectViewProjectNode" />
+              <item name="tesi_funneling" type="462c0819:PsiDirectoryNode" />
+              <item name="src" type="462c0819:PsiDirectoryNode" />
+              <item name="util" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="WebServerToolWindowFactoryState" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.highlight.mappings" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.highlight.symlinks" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.date" value="false" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.permissions" value="false" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.size" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/src/results/results.csv" />
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+  </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/src/util" />
+      <recent name="$PROJECT_DIR$/src/data" />
+      <recent name="$PROJECT_DIR$/src" />
+      <recent name="$PROJECT_DIR$/src/learning" />
+    </key>
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="RunManager" selected="Python.last_test">
+    <configuration name="funneling_poly" type="PythonConfigurationType" factoryName="Python" temporary="true">
+      <module name="tesi_funneling" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/src" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/funneling_poly.py" />
+      <option name="PARAMETERS" value="-d &quot;pickles/jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.pickle&quot; -w /storage/andrea/FUNNELING/embeddings/" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="last_test" type="PythonConfigurationType" factoryName="Python">
+      <module name="tesi_funneling" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/src" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/FPEC_andrea.py" />
+      <option name="PARAMETERS" value="-d &quot;pickles/jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.pickle&quot; -w /storage/andrea/FUNNELING/embeddings/ -e unsupervised" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="scratch" type="PythonConfigurationType" factoryName="Python" temporary="true">
+      <module name="tesi_funneling" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/src" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/scratch.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <list>
+      <item itemvalue="Python.last_test" />
+      <item itemvalue="Python.funneling_poly" />
+      <item itemvalue="Python.scratch" />
+    </list>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.scratch" />
+        <item itemvalue="Python.funneling_poly" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="f02472ae-5920-4b7a-abc6-0eb6f03dd03f" name="Default Changelist" comment="" />
+      <created>1574680487463</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1574680487463</updated>
+      <workItem from="1574680491429" duration="18756000" />
+      <workItem from="1574705313406" duration="1369000" />
+      <workItem from="1574758627235" duration="18313000" />
+      <workItem from="1574845439127" duration="15307000" />
+      <workItem from="1574870087360" duration="629000" />
+      <workItem from="1574871032651" duration="671000" />
+      <workItem from="1574873488200" duration="225000" />
+      <workItem from="1574876908618" duration="140000" />
+      <workItem from="1574877826026" duration="560000" />
+      <workItem from="1574938635317" duration="14980000" />
+      <workItem from="1574958501259" duration="1736000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TimeTrackingManager">
+    <option name="totallyTimeSpent" value="72686000" />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="0" y="28" width="1920" height="1052" extended-state="6" />
+    <layout>
+      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.15544872" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info id="Favorites" order="2" side_tool="true" />
+      <window_info anchor="bottom" id="File Transfer" order="0" weight="0.3297414" />
+      <window_info anchor="bottom" id="Message" order="1" />
+      <window_info anchor="bottom" id="Find" order="2" weight="0.3297414" />
+      <window_info anchor="bottom" id="Run" order="3" weight="0.53556037" />
+      <window_info anchor="bottom" id="Debug" order="4" weight="0.5538793" />
+      <window_info anchor="bottom" id="Cvs" order="5" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="6" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="7" />
+      <window_info anchor="bottom" id="Docker" order="8" show_stripe_button="false" />
+      <window_info anchor="bottom" id="Version Control" order="9" />
+      <window_info anchor="bottom" id="Database Changes" order="10" />
+      <window_info anchor="bottom" id="Event Log" order="11" side_tool="true" weight="0.3297414" />
+      <window_info anchor="bottom" id="Terminal" order="12" weight="0.42456895" />
+      <window_info anchor="bottom" id="Python Console" order="13" />
+      <window_info anchor="right" id="Remote Host" order="0" weight="0.32959402" />
+      <window_info anchor="right" id="Commander" order="1" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="2" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="3" weight="0.25" />
+      <window_info anchor="right" id="SciView" order="4" weight="0.5918803" />
+      <window_info anchor="right" id="Database" order="5" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/src/data/embeddings.py</url>
+          <line>162</line>
+          <option name="timeStamp" value="1" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/src/learning/learners.py</url>
+          <line>566</line>
+          <option name="timeStamp" value="2" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/src/data/reader/__init__.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$USER_HOME$/.PyCharm2018.3/system/python_stubs/1539147038/time.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="152">
+          <caret line="308" column="4" selection-start-line="308" selection-start-column="4" selection-end-line="308" selection-end-column="4" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/utils/validation.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="440">
+          <caret line="950" selection-start-line="950" selection-end-line="950" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/scipy/sparse/base.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="746">
+          <caret line="1218" selection-start-line="1218" selection-end-line="1218" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/typeshed/stdlib/2and3/time.pyi">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$USER_HOME$/.PyCharm2018.3/system/python_stubs/1539147038/torch/_C/_TensorBase.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="325">
+          <caret line="887" column="8" selection-start-line="887" selection-start-column="8" selection-end-line="887" selection-end-column="8" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3.7/multiprocessing/connection.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="183">
+          <caret line="368" selection-start-line="368" selection-end-line="368" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/svm/base.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="201">
+          <caret line="292" selection-start-line="292" selection-end-line="292" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3.7/selectors.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="167">
+          <caret line="417" selection-start-line="417" selection-end-line="417" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/scratch.py" />
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/externals/joblib/externals/loky/process_executor.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="191">
+          <caret line="271" selection-start-line="271" selection-end-line="271" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/externals/joblib/_parallel_backends.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="191">
+          <caret line="566" selection-start-line="566" selection-end-line="566" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3.7/concurrent/futures/_base.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="101">
+          <caret line="383" selection-start-line="383" selection-end-line="383" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="182">
+          <caret line="224" selection-start-line="224" selection-end-line="224" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/svm/classes.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="20">
+          <caret line="604" selection-start-line="604" selection-end-line="604" />
+          <folding>
+            <element signature="e#18103#25253#1" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="166">
+          <caret line="702" selection-start-line="702" selection-end-line="702" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3.7/pickle.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="166">
+          <caret line="503" selection-start-line="503" selection-end-line="503" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/torch/nn/modules/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="216">
+          <caret line="12" selection-start-line="12" selection-end-line="12" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="139">
+          <caret line="1486" selection-start-line="1486" selection-end-line="1486" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydev_bundle/pydev_monkey.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="180">
+          <caret line="12" column="38" lean-forward="true" selection-start-line="12" selection-end-line="13" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/data/tsr_function__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="54">
+          <caret line="3" selection-start-line="3" selection-end-line="3" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/data/supervised.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="620">
+          <caret line="67" column="12" selection-start-line="67" selection-start-column="11" selection-end-line="67" selection-end-column="12" />
+          <folding>
+            <element signature="e#0#99#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/data/embeddings_fasttext.py" />
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/torch/__init__.pyi">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="299">
+          <caret line="491" column="8" selection-start-line="491" selection-start-column="8" selection-end-line="491" selection-end-column="8" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/multiclass.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="217">
+          <caret line="180" selection-start-line="180" selection-end-line="180" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/sklearn/model_selection/_search.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="14">
+          <caret line="366" selection-start-line="366" selection-end-line="366" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3.7/codecs.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="174">
+          <caret line="309" selection-start-line="309" selection-end-line="309" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/pandas/core/series.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="160">
+          <caret line="303" selection-start-line="303" selection-end-line="303" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/transformers/clesa.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="72">
+          <caret line="4" selection-start-line="4" selection-end-line="4" />
+          <folding>
+            <element signature="e#0#18#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/transformers/riboc.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state>
+          <folding>
+            <element signature="e#0#11#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/util/evaluation.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="18">
+          <caret line="1" column="36" selection-start-line="1" selection-start-column="36" selection-end-line="1" selection-end-column="36" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/util/file.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state>
+          <folding>
+            <element signature="e#0#32#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/util/disable_sklearn_warnings.py" />
+    <entry file="file://$PROJECT_DIR$/src/transformers/dci.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="159">
+          <caret line="9" selection-start-line="9" selection-end-line="9" />
+          <folding>
+            <element signature="e#0#18#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/util/results.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="594">
+          <caret line="33" lean-forward="true" selection-start-line="33" selection-end-line="33" />
+          <folding>
+            <element signature="e#0#9#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/dataset_builder.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1314">
+          <caret line="73" selection-start-line="73" selection-end-line="73" />
+          <folding>
+            <element signature="e#0#32#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/util/metrics.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/funneling_poly.py" />
+    <entry file="file://$PROJECT_DIR$/src/learning/learners.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="2063">
+          <caret line="517" selection-start-line="517" selection-end-line="517" />
+          <folding>
+            <element signature="e#0#18#0" expanded="true" />
+            <element signature="e#23965#24743#0" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/data/embeddings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="462">
+          <caret line="162" selection-start-line="162" selection-end-line="162" />
+          <folding>
+            <element signature="e#0#9#0" expanded="true" />
+            <element signature="e#222#778#0" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/FPEC_andrea.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="119">
+          <caret line="99" column="40" lean-forward="true" selection-start-line="74" selection-start-column="6" selection-end-line="99" selection-end-column="40" />
+          <folding>
+            <element signature="e#0#27#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../word-class-embeddings/src/venv/lib/python3.7/site-packages/torchtext/vocab.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="182">
+          <caret line="321" selection-start-line="321" selection-end-line="321" />
+          <folding>
+            <element signature="e#13891#17737#0" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>
\ No newline at end of file
diff --git a/src/.gitignore b/src/.gitignore
new file mode 100644
index 0000000..b9a5796
--- /dev/null
+++ b/src/.gitignore
@@ -0,0 +1 @@
+*.idea
\ No newline at end of file
diff --git a/src/FPEC_andrea.py b/src/FPEC_andrea.py
new file mode 100644
index 0000000..7345d89
--- /dev/null
+++ b/src/FPEC_andrea.py
@@ -0,0 +1,151 @@
+from sklearn.svm import SVC
+import os, sys
+from dataset_builder import MultilingualDataset
+from learning.learners import *
+from util.evaluation import *
+from optparse import OptionParser
+from util.file import exists
+from util.results import PolylingualClassificationResults
+
+
+parser = OptionParser()
+
+parser.add_option("-d", "--dataset", dest="dataset",
+                  help="Path to the multilingual dataset processed and stored in .pickle format")
+
+parser.add_option("-o", "--output", dest="output",
+                  help="Result file", type=str,  default='./results/results.csv')
+
+parser.add_option("-e", "--mode-embed", dest="mode_embed",
+                  help="Set the embedding to be used [none, pretrained, supervised, both]", type=str, default='none')
+
+parser.add_option("-w", "--we-path", dest="we_path",
+                  help="Path to the polylingual word embeddings", default='/home/andreapdr/CLESA/embeddings')
+
+parser.add_option("-s", "--set_c", dest="set_c",type=float,
+                  help="Set the C parameter", default=1)
+
+parser.add_option("-c", "--optimc", dest="optimc", action='store_true',
+                  help="Optimices hyperparameters", default=False)
+
+parser.add_option("-j", "--n_jobs", dest="n_jobs",type=int,
+                  help="Number of parallel jobs (default is -1, all)", default=-1)
+
+
+def get_learner(calibrate=False, kernel='linear'):
+    return SVC(kernel=kernel, probability=calibrate, cache_size=1000, C=op.set_c, random_state=1)
+
+
+def get_params(dense=False):    # TODO kernel function could be usefull for meta-classifier
+    if not op.optimc:
+        return None
+    c_range = [1e4, 1e3, 1e2, 1e1, 1, 1e-1]
+    kernel = 'rbf' if dense else 'linear'
+    return [{'kernel': [kernel], 'C': c_range, 'gamma':['auto']}]
+
+#######################################################################################################################
+
+
+if __name__ == '__main__':
+
+    (op, args) = parser.parse_args()
+
+    assert exists(op.dataset), 'Unable to find file '+str(op.dataset)
+    assert not (op.set_c != 1. and op.optimc), 'Parameter C cannot be defined along with optim_c option'
+
+    dataset_file = os.path.basename(op.dataset)
+
+    results = PolylingualClassificationResults(op.output)
+
+    data = MultilingualDataset.load(op.dataset)
+    data.show_dimensions()
+
+    lXtr, lytr = data.training()
+    lXte, lyte = data.test()
+
+    print(lXtr.keys())
+
+    small_lXtr = dict()
+    small_lytr = dict()
+    small_lXte = dict()
+    small_lyte = dict()
+
+    small_lXtr['da'] = lXtr['da'][:50]
+    small_lytr['da'] = lytr['da'][:50]
+    # small_lXtr['en'] = lXtr['en'][:50]
+    # small_lytr['en'] = lytr['en'][:50]
+    # small_lXtr['fr'] = lXtr['fr'][:50]
+    # small_lytr['fr'] = lytr['fr'][:50]
+    # small_lXte['da'] = lXte['da'][:50]
+    # small_lyte['da'] = lyte['da'][:50]
+    # small_lXte['en'] = lXte['en'][:50]
+    # small_lyte['en'] = lyte['en'][:50]
+    # small_lXte['fr'] = lXte['fr'][:50]
+    # small_lyte['fr'] = lyte['fr'][:50]
+    # small_lXtr['it'] = lXtr['it'][:50]
+    # small_lytr['it'] = lytr['it'][:50]
+    # small_lXtr['es'] = lXtr['es'][:50]
+    # small_lytr['es'] = lytr['es'][:50]
+    # small_lXtr['de'] = lXtr['de'][:50]
+    # small_lytr['de'] = lytr['de'][:50]
+    # small_lXtr['pt'] = lXtr['pt'][:50]
+    # small_lytr['pt'] = lytr['pt'][:50]
+    # small_lXtr['nl'] = lXtr['de'][:50]
+    # small_lytr['nl'] = lytr['de'][:50]
+    # small_lXtr['fi'] = lXtr['fi'][:50]
+    # small_lytr['fi'] = lytr['fi'][:50]
+    # small_lXtr['hu'] = lXtr['hu'][:50]
+    # small_lytr['hu'] = lytr['hu'][:50]
+    # small_lXtr['sv'] = lXtr['sv'][:50]
+    # small_lytr['sv'] = lytr['sv'][:50]
+
+    if op.set_c != -1:
+        meta_parameters = None
+    else:
+        meta_parameters = [{'C': [1e3, 1e2, 1e1, 1, 1e-1]}]
+
+    # Embeddings and WCE config
+    _available_mode = ['none', 'unsupervised', 'supervised', 'both']
+    assert op.mode_embed in _available_mode , f'{op.mode_embed} not in {_available_mode}'
+
+    if op.mode_embed == 'none':
+        config = {'unsupervised': False,
+                    'supervised': False}
+        _config_id = 'None'
+    elif op.mode_embed == 'unsupervised':
+        config = {'unsupervised': True,
+                  'supervised': False}
+        _config_id = 'M'
+    elif op.mode_embed == 'supervised':
+        config = {'unsupervised': False,
+                  'supervised': True}
+        _config_id = 'F'
+    elif op.mode_embed == 'both':
+        config = {'unsupervised': True,
+                  'supervised': True}
+        _config_id = 'M_and_F'
+
+    result_id = dataset_file + 'PolyEmbedd_andrea_' + _config_id + ('_optimC' if op.optimc else '')
+
+    print(f'### PolyEmbedd_andrea_{_config_id}\n')
+    classifier = AndreaCLF(op.we_path,
+                           config,
+                           first_tier_learner=get_learner(calibrate=True),
+                           meta_learner=get_learner(calibrate=False),
+                           first_tier_parameters=get_params(dense=True),
+                           meta_parameters=get_params(dense=True),
+                           n_jobs=op.n_jobs)
+
+    print('# Fitting ...')
+    classifier.fit(small_lXtr, small_lytr)
+
+    print('# Evaluating ...')
+    l_eval = evaluate_method(classifier, lXte, lyte)
+
+    metrics = []
+    for lang in lXte.keys():
+        macrof1, microf1, macrok, microk = l_eval[lang]
+        metrics.append([macrof1, microf1, macrok, microk])
+        print('Lang %s: macro-F1=%.3f micro-F1=%.3f' % (lang, macrof1, microf1))
+        results.add_row(result_id, 'PolyEmbed_andrea', 'svm', _config_id, op.optimc, 'test_datasetname', 'not_binary', 'not_ablation', classifier.time, lang, macrof1, microf1, macrok, microk, 'nope')
+    print('Averages: MF1, mF1, MK, mK', np.mean(np.array(metrics), axis=0))
diff --git a/src/__pycache__/dataset_builder.cpython-37.pyc b/src/__pycache__/dataset_builder.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..54ba91e8a263f92220ef40cc06e1fd3449d69e77
GIT binary patch
literal 24002
zcmeHvX^<S(ecyD?Jv%#lUjPU+!3#_6E<lhJB?zGjk`f7vgcuSQr4gy|U|#R;Eaq6e
z?ge&NJr$cvS)fv|>Bw=M(~BHR?40E)JC5TxmHT#_d`Kys%7;`c6`?9gx$?oO(wF4-
z`@im<nZ<&qB<F*7YhJ&8{rVmM_kaKU^|Pa+r4;@?75=9;pXsJj|CKlKUm2Na@bmtM
zl}f3UvRbLORkhl7)wXzVx6<uQH6!_SE8EUhbM1UJFZY>Np<S#NC7*4TF0+2FHPS9u
z%k9zXXuDFa$g_NFtUX>GZ%<Sw+LP5uDJ!(5+SAqP_Dpp~?u)J2_FQ$YJzt%d`%-J6
zx`6gdts~W=xQ?`rwePFmC(p{Q<LwjG6Hz^<>b#IrqpI?DN>%pk>iud=jlZ3$o>UWR
z64wXRl$yr%!L^i{S+&)yntR)-KBS&f^J)RPht&~v6we<~$JBkeo>HGuj=CRD7S*TK
z1L{HK9(6yV9=dA3Yw7RGn3k$b?y|eIv9f2Whgt4D@1ItW>{)MFELEo#Qq?n(TSV?L
z$vukPS;>6@xyRkITEg`Sb)x#@cE|lB+FjnWpHIE{<JG6!PpK96sXeTKyXc;AA9tT{
zKaCov)tNmTJ)T*!+$Ysz>a6>edi)mPkNQtU^&fT5x{u-axLZ+Au32*_byA&lpG3`1
zvgW(iwYOgfR9{Oip1VUFShT~;Mz`4sbMAK2^SwnYEP8%-^F~)I4-dyKuQt`{7u|;6
z)y-X3hhxumw>o~m0543vyw&oXFV(NOt!JB^y3|9l_2^<cEVSyKwVLPpVfoTamp@;7
z>B6NKUcT^3c;BnfeetZ^y;ghu;^h}>^7OUKpMCk|i<e%&i=|c9Z>-l)?XiLU*>J{`
zV~EX0ZN2I0x^ArRV5G;)gB#7O%}rM|>$NsV)oOOuw(6~LQrA1G+ph6(&8xRJTQ1(2
zV6U(FZu@fgs@swIO>b&<Q+FG#=halVvDJ1vzIO@J8TWO)*{S)i=hwW=R?|mWq3L;>
zb?vzg!ZQ9#Bl8S?-ftptQ&mf)s<yJKX=ST4Ff5}!p|UCmyvnM)Dxf5%imHTb-YuvR
zRen3QXH|>H3sxgvig0>FEeTdz)pCT@qiROY;?0VhQ}eivsReZe*Ku`J9m91(-KUP@
zI;l=52iGZezdDKQv|x6sI-?#^595ATJ)%zGI;R%Zqqxqi3~*#QTH6SU&)5Arma_h<
zpcN$te;1rVAUn!DcbcA4cT}_CH@iTG9jCkM2=K5L4wk_o44nkHPP3!jZC5#0b{t~M
zR(;KN8eQeCNFSF8_Wo<=AFXTG;WD`%RtIm^J;&d|l6unBy6gBmo33-}OG{r~Y66Xy
zoEz&dUip$4S-bA*=61h<I>#BkDP8u5cFJ8@TUp|4B2-wuva{^Fy6u>CcN(6qnfB1j
zm!p{v)NOHBI9IqWcAU=UN?q&v4rknKpj*JhUw5TXsVL)j>y7oKKlBZNVG^e47WODd
znhd9@0D<Nz_JQZD>Ta9uIoPUe=&=PTwp!S_n4W=U0}CgAyxRCZho3i&L|H+qmTIPu
z0$s86_N9=-p&YGgty6Ei;Y70BwWu7MAmh2M)n7{KBWUJ(pE<qWZM&y2XzkWdd)J$-
zt<BBuX{>m&wz}2nxJ0GXo^G6u8@{sDY$;c-Z0>}GTCLe>`nB2uhm}E+vU1k$F`+NR
zIJgp5z0@eM9{zacJ2I!^$OLwf4sxsMAQNPR{AzmD4pJMbz&Z~`g&A7d)p{QF77Lmk
zh8bdCnEsNl!}OP#e93p2bHkjkYghwZU0!9XCOOI%({-ipL&hUWQ`YX>VFN~;hCD|V
zUUT@#P){Ngr1n#{Q&!5i_Ay959azfRvwdqLb31L}saX*F#+AivSoXTw2cWoVFeK2y
zt1UOosIB&9nCWi1opAici!Z$R+1D=DUU~Hkm%s4b7hYPlqZzI?TkgTpXT17#*TEn>
z4k=|7tXXS!{;(;q;?Nw@s4sKi+6O=(&f;oL&jQd^U@c(WF{te@OYnLI;_-NzpuGpj
zGg7OqxjvuNY7e2^Xjo;dWbK|fY*+*JlhJWx*?4rvknvOdmTw!l-nM#nVDG01@xYE?
ztFJ6(^dqPVJm}VyUPM+uj0DUm-Dr9G6Y`8`dtk)+F+6z`KQmH$93VU#fNVG!B8Qg@
z@fqA49HL<tfxVI5&)iNu52nyds|;DlP+108o>}EIvX}H3w3F;|?gp;uYxsGb3itP$
z$KSAf)*JR4>7I@MbadrjvA$(r1KICSZ!r_vD`zxuJG5$=nA(4+iTmFxohPAhblaOx
zKaP9Z0=d-gnZp)})93zKIp69w>Mif-mHx{gYS78x4<EFCf&=BK^^;5<U_ul%3CFEJ
ziG<an0WW4{?PV#=(vvcTS)!qbhiCZ?F+g*c1`>xf3wbA+BNxIG(gb{%#{c1DEUunR
ze+VmHLL=LsBh}ishyibYYGCcsvUZssu0U)EhWd#Zsd1mdeL2V&B)?XM&==9XjO;-D
z6sIwbq)$(p5PC3G(n13~5>W#|hq(g~Bto$jPqYi`L937|;J51>w)BHEtHI65KVJEJ
z4nL1`#r8~HE$VYVay|&4c6@Bpz*@7gWv$M%lxI->gehk!$xbaB*z)F!p!um<E}m;X
zEH%Am$Mfr*21t3c<A-)<Gt3GX#y$r(4adNU-8HaG*Q;G`x;Mf~EQFZ*#k8TFrGCAz
zm{d6cl&}E#M4CCzIgzuHpq8u(ekCw7P=-U<S^Qdp2jtKRY}k$=u1J9ivfIJGAiTH8
zt}-p~D{wPey5VIC3$f_+IW!{J4i0tb7~+Z_Wei|I7=wF{F=Qz5T^#8DQ1-$~clAve
zc-)KMNUee?D*Nri&1669S)h99OUX+y!JFbsOGt#^Jvh8+dO)a~sfbf$fG~q(4-;6%
zdte298j@^-q1h!=<YjhdIQs6=VYtsFI*jCjD|2G96$hJ+^U}lfsKKofuiT-KV-E=s
zV$Au_kR$0i6mmV%TVWSNkQ-(ghe1xLuY+deDG6>Skb5Pb7Z}F9Am@H$$i0BpzYzBT
zoWB?3+>Z}AGFgEf*@WQSqeD0cMmq>OV4oGFZ-aCVFkD~*F)u8D&Ikn=!a;%L2|Pa>
zlCR;(d*dJo#7BnYXVLnraS!)|BzpMxkW6s!AS49`A+CIINa`OYxCS?}0P%j74wA|Q
z9lsb)@t)`i3ipwL_##^WvABo#0OCC)><95G*dG)%u)kwulq&UgDm2!rO*uONDHOJ6
zU(!E<y5J4@^}1L0eI4B!HfDYK0Lv0EeTf4j7N@|aChf!dLyVZoDwF6XJZflwlN=a0
zb_J41%E+&fUV-o;Z0l$6A|#E^<0APmNBO{2LJz~tRd)xrC&SISCk~F7Wnab5$X6I~
z!LC@l3y0EP>Kw^{W77B>UnVUYWU31K%0g<xRsv7370v2*m_P>L9xxj;>#f%}@5SjV
zovn7QFDbZqE2i8h4;$Zb^<<zF97phzfo>zStp}t)HAGx8AkNudu9p`D%!UG%*JrPl
zp+v&y1Y}_@8|5;}-p`_*y^+Jz^CUvZWuv<s?`~)JEGv~ri~s}4^zI`s0g}$wdP@#?
z%^%`*C%QsQj_!GdGMz9rftRdunBCNrzH+NwO>+sGH2c%&-UIR`iCmbu_&o5|VJ+@g
zxxY(s9rM<^H)^U06P*VWhqsQJi7;AD*%Z1TI1K0q-b<ifK?5W}0`=cT#-K+Jvgif~
zIMTGTZh@E|xKAIr&usq!Rx7ig6`B2;Sgow+TUeuB0jpK)m3kwvNu>63vRWJ0o<|FL
z+!uoUZOAw(jb6%0St%(Sg}73FDg`2#4n}dMZKJI0s~MG<hA`gC_(iOBb}ki^X+D;_
z9PUPWckTJtQKItD737&J1Zn2PPLf$oV{aB|>C7sc&jkBv-l@YdtjS1sbIT$k?q*>E
z^Zeanz150J3X(<PVoCpDRMixJ!z>KOT~B|81;a=YroFAUAb=)gS**xruiBmGs)c$3
zR@a)Zlbl;E!0>9U2cp!8Kt?>#mzg+Bh$fo!wOWE*@H*ay*|1tPc?|3T4G9+n38J04
zQ)9h)%Y~ZYb<l`MRX1f9ta15&&@$*W4f&mA6>F06yuG`8PxLyh<wW%-;p9&cjp~yq
zF^P(1g*jsmjEUeIsB&-uC8`Xj+FN)WQ<>w3jwq=LyB^dA{1#|!qfxI`3rn?HyQ{WZ
z%$IAmYcQBcPx3Vy!D}^56N)B@*A%PuX(ZuDqgD63w(GBVl~J5O#b?wDHD{)&acGMD
zno_K$KxXJDd7)W-A@71Q<i7eVOx|SD*JAoI;|v~5;OEiUQpjcOLZ*~1WDD7HuADDq
zOQk}3;6GQ$<1Sa2)_gD7<Z0Yw@DtddLdH++rFz!(d_V<zobm;vGke3n9g#s=<3!Pb
z!j%Ey!k3ZT+Jqg*)fe!(ei{jEM0o&86iN}`Bxk9CX;91vCt%sC)nSpn?nZ0(4vHf_
zmIF?WTbg7n!4!@(AxUR&LlqdT)ZRWR<2?f9pLVVEMtaZMhc<H?PlI$VeXY`0`LkgW
zw&7inN!OR*>IPbp(T8P_Wf+f*l^D_zIaYP-s_SkJu(4_m=**Yb_+Ec$WKPpCDx!(n
zm7Re%eg+jIfr9F<zKZ)~3CL2DB_NVeun~R#Y03+5aamV0`YZqs2jl|p({nV6rc{P^
za}am>&rDtx2=*`1DWP27vzG7tU;JT8=UB#73e(u(SV8K_VRpxb=jPyKDz%!F*Zgh`
z^?nX-MJ%Ocm91m=>1!yv+{h#4-z<LI!Gcbo!^{Mo@<AR*l%d_if<n+M_%@KN7^H$?
zEgclGQ2C84pKbqcU;*vE+bj7wW%;1ZL5XujE^Tr_R{uEevgWP>@gyj%W@b?b^u1u}
z|8xpmWu#Z`jiN-}lafkrtT!Iy^lyOb({h;gOF@aQr(3o^vS0Q`*(P{P{#Ld(0a-1o
zr3$ywy@_CAzY>hV!ZL~6SWrQ33S;@-awnsK0#gV|=Tn&LSWw3EX~~TtH-kBj2c=*#
zm=30bnP4;+zYVAHwI^N&1okJy@LJrTlxr4l)u)4EP@q*KC_;?IZ?RPWFW9N#8hMr;
z0(f}f*+~2zu^Z-w+gw|x3rL(WpxzE73`bljv5jWMc{O$wiKoZ7k7z%Kt(@)?I#lG1
zF1pZ5(r0tUh2LkWon>#+ZNSyiFnw)1Etrhp%5mTwg2Ryn-&u9*{uVq<@D4R1|J6A-
z2<z?GHzp+aoXmwJHuhIV&JgDmaZ?5my(V1m9J(@opLkB*-8EzD4GFe)4rAPOzQ*e3
zaKFs=dyy9`_AeQalK9P>Q@*?HFF6s@!n~H?a~c{5IsOl*X<=dl7>qlfDWN0ou*flX
z7i7efYvvly&^guJ6rW=Y&ba0(RzVibRZ9+Bx*MQz4&)SvZxWMa@rYGJtL8cMNcn<C
ze3*cIa9JQOL{8#!gevg{UN@)H^&P+K)HgR<eP3oDZwO?JBz~3dKTOLw+u@FEcOim^
zzt4l!xsAus3TPKd)j0=5JB+zI7guFUvN{fQKIe+-b_ORu<cw{-E(_oCy6BSa5<EB7
zn{YZtJ|&sxpfi?Wh^C93mU!5c-53M1snH~XPX3kvu0Q`_I7>IXpMkh0-gZLcA^3(V
zw;{hIH$Xbc$dYkq$E)F-f5>}U2z{8}>NM#tz5{xZx(v#mzwYWQUC)KDY!v&cxe1Qj
zpc_BT0jM>#3X}KN75Edu0BZhfSo9O`e~HUcGr<pOy@dw#btX5ETw0vcr21icwdspY
zq38IF;tH%bl!ahI7I{`vh78lY%}q__ttqr<a=b7rZEJFja1`xOPVIQqiZF~gYXq%C
zAX=O?9f^5pGzO3?H(rYOWjHdhkHTWIJwnRE5sbrhyLF{%w_Zo&%|LWS?94Teo;;kL
zq(>Z3B&3L(9p>Tvj2^_&i%8sI)@P827O~vnR7ExyiTr@9a74Vl;=GKAukoIPaQ`A+
z^e8~4#&TBK&e%Da%jxvXSZC}>7$q`dNT0SVHcjanlu!ge0wKMCx&>SRQM}s_BL*^*
z`4YZ-Ddx-2dV^dI0u#6~c{KDV!=r;-pC_+@UvGa?`0`u5V#1e;31606P+Uz9aAA}Q
z&xK|O&5c|*E*b5C4})|5JhZV=P~5Y@>x<xs%#8+xeeh&T0vN#vxL_$5!Lv$G4#r}R
z=|jUa{2P3EJQ&BQ3nDvoWQ?ma##~S|qY02(NWDRmt&9a%GlKDb@O$v)3A8X8j3PG)
z2#gAkpOPGT{B$&q3Gi)l?|2;F0q4%`SA=8dz@PK=ul_*!GfC<D@Mw5;8m>@u2AHh7
zZE|D>p&X=Fi1Bfqovm%Zz6nyGhH1QshWG={{C-S-fTDZ|&fKRrf505UbtobNBR>jr
zeDWU5QG^6COAvn%REF822JO=xlFKk$4l}Xchf79eCcA|UWa8WIZeww5fWwjN>0KtG
zntzQsnr(#5!7z#3Bof9D`VgrN`x3_YNBE44Y;mT~!}L$E^e36na-rX5BKD!LBUc57
z6E*|=tKPV3EQ3GAD&OGKES_F`Uf*N^b!QZ~Echl1J^%+=FdS$jp$rKF`VLy~K8Yk%
z0qX%vDcfK}U^ORU?U@9_A=@cgM__9%f%TNF`KW|t1xw$;n+-B~{!QUW_CuM(`xRk7
zPza%Hy<xwTe#gF<-iM*~rp@%K?LF+nu*m39KVzOi!_W9x1*blgyd13kP~UFZG$i(N
zR1zhAFcoAN6tc&17#6{FvOz&idoUvgIb-Bw=+hi(fk`0RaKGe($?QQbm6o##><1+<
z`Kn^{1a<`UnN=lul4o5opb^9?LeZ1{&?DHDv<p@?3RXH2j6hi|8dfR_aC!SzIJ!4~
z`5nt2Q)M;kkMB?TlQ&^whr(I8mENDaoq8wbPk+Pu1_HAX-<W$F$|TztHYzMjp4xai
ze&FeNZ=yFT4&e!TGAh9_qf#;{CF4>uj&aS1>IzjBxd~x^<^9>4mN41zU?La`CWX=d
z9vEDCe~u<`Mizs`O<meO0u}?yGPEb^iS$cAKy|Job$3l9D)$ljO{Bqj5QZqh(!d~5
zqKVOi_C8|AYs3g*`x=ZMus6Xeg3yJu&IH!aO85<4wcU;wCSd=g8#=UF2kaCGyP={a
zVK)oFuGshY6hwMNGQhf|8`mizn*Cj4XkaYm4S2AUHg=C(h=q2<D+<YC<n8IRu=wvj
zjJ6&-hnx}4mn0{WIg0u!qe$bslc;=FDkr@Ufq)xAKtPHiASe+88!0n+Gi^(DeLVX#
zS~@U){m1b3$^OK(#=VfdA&R$y(xKi-l(;oLf>nB7GQHo#Hslg)ST_+Iu)rWT-oY5S
z0rMZ6Z0W)AWsIb3H&uwxhHYO=Y0+@SMuxmts!yUDGjy8DWYo+3QUBnECWb5h$Jyrv
zB!>;{fxCyca*v@=rWUACnir@|_JFkesheVqyqzHK_bdmlRbu*3j|h~mBOG-H?tlZp
zIQ?;Z)IZP84(n6@2|PK_-I;!Oi}QxTQRC2;kc5>BP~bG}2cOlttBs-bRX#2=8D&B<
zrWj~7aoDJNd5MU})cS0a026&r_m~rHuVNTa0tjK7!XF=G9h%h_r^1nlj%?Qb^{~Q?
zXLuo_YUTdyz<s{E<+GykOGdThJs;_V&`8lp{OVy3&k7fYx24q*ol=tv!<=n`xeZSh
zh3*h%i@+BqR)Mu(yr7k!y%>id_y7cHf~z8o7p!$2Y?m&UdC~65(DII0Cs9InOapn&
znzfHx$Au*ucC7y*#@it9`A5o{Fy5a+%Z#ctjF(IjdJb4Fw641tuXz$N%7pO-h_{3<
zt^}rAq~8?w(~JW1Q`uX2uwCkU=1CfUxEw91C<XIU1yy`I1-)+!`dl8G-UxEg_Nem-
zb1gsz9D$t|nxLd*Fj~WUOJJzf@<xL407ISF{u@CN4E5V!sAR{m-tJfYv529Lf}xHJ
zL!J1Bg*d5-8Us5WN!aORZ>l%#n`h(lOjxF{+L_*LZ!Th{d}~rzr(v&?Qg=p5W-zA7
zU<xdBN^-Nv&B7Wy1!jtJSb1lmFU|=w{cSMQiGApjACH-W73L6K4ug^y_80R8(~HE-
znqdAWmOEr<Ud?zR!cpe2C0zLTS}H8i`ida<Ryf-3!h*S~-D_Lm<zW^!mj+Bk#z*~a
zG^5#;{>w~$2}$CDAvsI<#>2Sr;j7q?xJ1U~Qj~gUAUch1nS*g@Y=XRBh$y$a5Yc0X
z^{CVw$mj;#8el@gcKUKat%f*`k$4V>ofX0!5!cY#rnL(JwLD;RfYg1yPwIAGPB!6&
zZoIEl(0AuF)nT#%VAjMG!Xjd0wcqWwAoDcVA@jhXMKOcL^(zQT&%vCzg^*HPoJt74
zj3VbjuKy+)koe)SJlNeu4w`^U(Rcspk38(%hcM9(AhCQvsUhKibim2dojNEr#Ii!P
zn3=g$EHPlVe+{nzYC<0qo~-{mN(YzI2o6T{Ik1@3fyL~Lcpqdj`(omdu<`poqp$R5
zbojvDPt^EujQ@=vZg+3?yMy9OQ>0Gh4E+ndr8`n|=x_}2$uybQ+%4U`-fe_q<{%qZ
zx8At6h1iQk)Oc?RL;oqZd6-x+um3b2g`<KnB%LBnjEJIQ+?`=b?8xH!G~{%%+iEs;
z!ZD$m&@<ZI4&A%@&#@EX@51S6lh&(DeuBwcOn#EdlT3b<3B?Kh7nq28{@$X5Js?UP
z5FqZ2?&k<@0yq^55Ox{HVq-prIFX^D__%2Fu?TStzFcbckqGhkIJ`EJJ2ye28%4H-
zKaPO;=kW8M1{GJ(qm>PJb%Ac^UO~b8jvyN36}CL`2)jp~0XY2SR1TWmvnr1_a(n{!
zZ2`}Vc(+7XH_W}fFF`|xh%itxA|>U4lCqSH4wQ^aiG<ieEUC!d7#-sfPsSj$jQfmk
zPQesC4h?z&qnQ+;1-f(eBoj>X9H9s;6cDm%CbBq8KoG*oLY$RNfysl-GiV5=<5{qI
z=E&@)BsT>C2!agG!2XV!=*<OrHGee&m+o6AACvNRSSBDk@noVpdMneLgV2L|Xl2eX
zLCl%+VPsavaNeo3G0O8!opF>`r1ZW$L^KS(5zL|OU-QQxj@cLsjns2{IS5_z1FcmK
zv^G$CFWZ|B=HZ-%Yg_Cb$7RNO1WZ})+8)kH1{3PULaJ%?7LcEc@<-UCI<aO23&D|K
zaxaS(;PRFhoTvo|a%kb4KhYnX)Pt!V7KI5|6z;Eo7m|cHzFvV-hQ}D`kc@2yVut(C
zSD#zHARB+hxjbN4fElsT+EO^%h{BVRD{XneM1InN-;058OROTDB-Q$c@Znkxwly}o
zoRN)RPduMLwuvaPXK~yY4njG$iH;9*1IDg*V^zNhhdAccz)4Yfkz;SwfUS|UrdzJz
z!%5odc9vuDChCPY&BS&fM?I6;aZQ<AWWFCFYUdnEh4jG^O1gf->4S5i(Nj$yZcL2w
z3PTERba)B}J_5Lm>2_G<l;J^6=FBGL1gZ4P*@<b#OWIg-JCJ!0hA=oV<AICG?eNG3
ze5g3K2QRQ|EJ&wuY7&Pbz0*kr1foE~!s*B%Z#?b2aF%nDvBNwz0mC-c#lcS)t<aEj
z3hrR#uEN)e>2(CtPA4@FwmQHN=xmN#Mud>Z7JcKHm0pEYuqQw;?hIY<Aa3Nq?Sp#|
zD1A2bemywv(Zvvo3^q^U-*7HAPrQM74tpd%ZEO78sKpaMJC4yUuw@+cQsTT{alTk@
zHI*r5m<Ny+;UIDt_#_bEq;!g9qiz!6dwhxrm~--qCPsPwNiMC_>2`qaT(9^M!Pam$
z6aRqY*SE15+dFt4j_#8!7pE<~egpBIHs$wzd&PMT$aFxcFlU2FXCe$coro7TgL1$H
z+4$`)wx_}&?`F#zgzN!1z~EjRv7zm)wu30<D~y0x1wFy;HR8g6X>brFWp>Mtv4~M-
zgh`ta9O|sTEhqKw)l0HF`baj|g>wohUx$O+dGgGeGea9kZ%G^l2JC_YA)J86vk_|&
zn{mx&0?ht)PHlPa>Q<{iR1JrO9KUX^uHryBOu}f-I}oLDsoOEsw^{`8F*013si9`I
zNMhA#`B!V9ecdxM5~Yr62HSj7&oVnx>vUr}hp<$Mjd0a&e#IR0-5ssrL^aP#%emj(
z$y%Sb$7SUe0w}&4H+LY0Q3*7Xzl5GdrM?vD)cV^j>1)7$g|}3L^)EB|t4Q8MSKeKG
z!B9Yg@etM&bGVh2l_9k{jb=qBo<yd9z>99}gDNycyuKFh;6{$w%tx9x1T&lqlh<yh
z%%Pe^`*6kL_e+QSxxP-Y`+2%ZK~icl^Md|9a4NR8rn_6445%cj9?-#)qqZoE&Hu&(
zhr^$M3CsZnu+ADEVwFYVqzo&kUHb#DcAxpU12DbX`X8`$Q39y2p-(9g$-Oc>jc;-O
zMz_a7?xhbi0P!r`Ig1WH2Xjb~K|B&h6BZSSD=oI3eux9%+U}kiz(v_A%U8(Iol~Ff
ztN}3V%Ou5%11PEA<!jEsYyB-XP;GHici9Q!KgGtae~r1n&4gy+>iy9-M&$D%*a@}z
z^=?x!6YpnDgx6zC#+f|AgfYnG$nm4hJ;sEgEGE+W7IKS6^{?X+R=BPE_Fn_{zr(7(
z!Q}5U`At?VfG`UI4dMUogc%0YAzZuLS#7RKpdHS|GX7C)Xkkg96y;^=CU0Vhj3wIi
zjF>rjbWe@1x)8H}F(rePp}sM|>3_)K?QwW=d{*XBogJ?40$5z0>SPkvREx`uO@wG>
zEJyOCmV=vUjac?Y)JaWeta9d)aO6_W<9}sXDMZAOD2Ac}8t3jf^7A;ZU4p7O%bZ<;
z<(aNtIC~|h{~cPs^Ia@sgGOflQK>E9$EEdNL5fdlWUl7CaUa3U;?otaP^GVCya!PR
z4e>1m&gvJT$sxaw!`psF;fOQzH~3n;7f}cP*taZ|>y@B6X6dhmzF1TR<nhGlkI0Kg
ziI%Rb67))FjL;w%-~<mZ>KJ{oJWyiv#nFKhqcc_pO2#5BQrc1DayRbh74$*qI?&@3
zG)8)&-%0f*pvT#2TFt;SoOq(&L0FMrf@5e>{LdvAhbJUAax-;P=GcSAsAds<Yz0%C
zvC*E?97}Ej25LTfV$9)rwIEMW51!~z=#c2okb3&QE$|JPylwg3O~H%<nrvR(C+${(
zX`br}vPL(brcQobow${O#t6f=vhb}Zj$cgN(D~+2a=(<!k_Vw=9-3c?>jRy29wVma
zCmLOVx(g|FGRS@%LF&9apW5zl1aGeP7J>x?vD@$3diq`anhl*WhY?H%3k)I)vh&pU
zkN6lJOW;dCGBm0O`eQ<`3+Gen!Ty*q3Y4%_9Oucahd57lvUfDnylwp*e=-;sUG%s7
zDQN#0^)SEg1I_!WKOLYR`abH<?9T>A{W;hM7NL31Y|P&_n)l59f|NeGN6mZSjo|3J
zX$_YHY!(><3$#_(qZ{N{?>=eoKD;?L)ZX;n^?S#HV~O^hSD%n|Ix^5|VK0a02u9<)
zpfxYmuYqrfeCXiso|?0GtodE3kFPkhpC7jJoJ=rMY$NAhq&S){7#*T}^1CmQ;>l4*
zFAsYThm}vlY5Drun6)L05utNnj0cB7e}13m+||-RGaOu^TRIX<59z+%aU#ZhSJT7n
zY`AGYlka#roZyGhkOzRk!F(UJ<bjrrrVJO7NN9|<Kd1=Bd~HBOfz+PB=OC1Y-Syi$
ztid6r*LknTV($NaRG$XsPRY%vB_Afdum2Wxge2_UdylyHfj)n5YMXQdN0S8DC#oAu
zjac}(2zePaZ(>q0Vz>|y5u}pUgrV!o7UES=#dT>(!b&<GT=wxEiw;meO#{GS<aPWU
zx^joD!<u4FAKYld<|H-)Xp?cR-HZM06^$j@%cn{99It`TKgH(|VC{kb4K@sph2Ob*
z#EY$WSb^Of-#mR*|9yxpyVE2W%V&*h>g#8BUc%0)7V)vWC|l}lqR>^LN`61}(R)Nv
z>7T~Xpi%q`E<>sul?eJo-Y1Gm)-Tpms2%B_Wg-G(B1K9_&VQ!R2<tvRmb7~+R!jgA
zA$!BBSzqt^|2bqzf9oDI{cru5CLl#oU$cG?H=<b|8`i9E+H>O9zK3?rB@soQfQ=)B
z;1i`wTjHt)qo;<VyWYd=`QQ74-F^IhmgRjlV_W|shNf>Lxl1!1?CAa0Kk4Yh>zRng
zg`bV)(VB;EO-5%oB%Alvw^mnSDCA=BkJt!Tfx+`*$OurUjoX>g0(3aXE@p$|>3+P|
zU2f&Wu>ODc!TZDP_J8%ez4XKE_xJk!{xG}!KmBe$gq2MHBP=<L9r_>Rs{aX-zJcTY
zEfq(NlbNTG<fyNFsC4TJqeMnnCUy*>m<i*)XMEeAWEJk~zCi*<h?<=bY{wAm#c|^_
z{xfXn&ocR^OlUL;E5m|RI65Q;>3_zGZ!`JlO#TIvf63%uG5Oa_{tc7gX7X>D>@)fE
zOx{9LjVukYTL?Fi!|1Rr$Q~pwrIi8!;G5M6>3Fb6tY=kozRIs(b;Wd{e}}D(_pKr3
z+n+I)Nl45A3&kKW`E?@1I+{sM@xyS(VWEJf0``W=fTcnr;AokE&B8>$O=k+0BX*2=
zdzw#D8QLgNLd(S6_KSIxPmBFRhv<u^{oABGwL2s$yO(e1F66Mj;l8fdOsw!{M80Wi
zSDkat!+>3T<-+9`pLWhuUKq%qjq|Tx{QSizFBauHf>$0oxkIi{EwByBA9rXm*>#@f
zX<o{JgPu7!Cdg-LPH%o2p?uB%fCS&wY_*z8fY)M9_sv{_6p5`%Fk|;b(rPRaC3=Z8
zDzX6L6u*NZ=>Nz>S|L^~T0AAuX4!8sL3cED|9j>oVd-U4d@js);frI)rOc8xpXvw=
z`YVmD-rVxcA-@wIzQ|eEot4NGUSo(bPyORNtP(EFCD4q)sm{bzhvZ`k^r#-Azh)(J
z&m8pE;ETU}nh1oNaS9S?Dq9>M^c{Ga_-EP~h-Iy?|D%J|5}nI99s7pLs5OqAU&@HY
zf5ga6u+vE<k01%Nz^Ba}`S?e5E}j`aE#bBhp++|tARzPvComh$&7BqV6<zrbq5fNJ
zMq*{SJ4~qUHRi+^N=hP;o?2oxXonNT)?Y;qXZm+M6OEezB3~EsS1l9sp1*MU0x`H+
zOb}SI27jxgg1&JjV^Jk)gjpDK*IYw)Xx%jtJAL6Xks!mudGqm%r-?b<D@ew%pZP0b
ze@`RF4}unOkCJp{8=s`e7Dg(cE`7SNSXeH-TKWn<U^Jg*C<=Exb~9~|cw-VVulO93
G_5T2j)het2

literal 0
HcmV?d00001

diff --git a/src/data/__init__.py b/src/data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/data/__pycache__/__init__.cpython-37.pyc b/src/data/__pycache__/__init__.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..12306d0a5175688d6267f5ca0c3560ab39d5cde0
GIT binary patch
literal 144
zcmZ?b<>g`kg8B8AVnOs{5CH>>K!yVl7qb9~6oz01O-8?!3`HPe1o6vKKO;XkRX;H=
zr6@H~zql+jr?j9TU%w=^I5R%2G%qhTCo?Zyzqlw_KP9mwQ9nLDGcU6wK3=b&@)n0p
RZhlH>PO2Tq$j?B`000k-BWnNv

literal 0
HcmV?d00001

diff --git a/src/data/__pycache__/embeddings.cpython-37.pyc b/src/data/__pycache__/embeddings.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..88eb734b2b0e4951155e0363aa37b8a4fa9d569b
GIT binary patch
literal 9387
zcmbtaO^h7Jb*`%J>FJrB9qta7OD;vrBU+?1iL@&Oj^fZXL5Zd$2g*t|>CkR_Gnt+0
zS@!JwvAXw%!=41z(BTpR4JA1kh#_R=62L%i33BE+$R(!$eaRswos*L;&iAT&dVaVH
zWMmdyU0q#2uU@_Py`SE9mzVPlzu)}FfA0MION{*&JxqQkDmQWYe}_Uc$y>~c@7&?~
zSvUsnqGh%-PKL+tEXP8d(aN@M$8P7GoNhN;`Sy~t#M%2yW~B9)Nz1i|hEu?ER@!*B
zhr(ILQ%>gbly@_DTg3B{Ea17|7VvyVF3aL$=B&swas~HQ$?ver>gS-3ZC7}d{S&ty
zbd_Hbk@3#Ex1wTg*AG;!9<<%yepjN$#-1Bg{ezyX4n5zMRkywCO6hg>{5nlC`58&Y
zzln;=XuFPh!Wvw1DIVL7;hNHr=3_SGP6l<oCDbi98*c}7`w26dlLLXdXzgR=f)KCc
z%G=Wwd9n)K1uk0EFQN$8kd6455$iCXZ(~ClO?6~ft6s+os#R>p^xaki_guAFZ`FLi
zTKzTq;@eyIyKQ%?){)AsZTW{@>!8=`ZUwIIRT~GLj@!a+w|rIKlC_|=HD$nN@1e4>
zc!8)Oav1M_?KHYKCnVHoH$e4UG?c_9AzLk>Ug0W-o36(Ntq|o8yY<@cL93=7p272E
z#jl^b;<WGoYfVFX?b6I*7fxLao$3<-AcoL}^v-rs*~6b;17lzwUkg~shy1&2XY;3G
zkQrE^6^i|=y82)(6rr`j2HDUU@h~&yoP{Q8S*kr)yT<}CGD2g_2ZpYhsF|S=nhg;j
zw;AQ!&OzH%wZM(6TCeAJWF$JhC|mPuO4S}#?8vCM{3vtORnm9zbo)K8Bi-XD?{^PW
z-K}~O^`NFm;;GKP$fRx&Z!VriR=IlYIpWP43MTTr#BC<H&GWp7E6@9vPUTiIw2~*$
z?9|$BwHoEC)pl1Nw5VRFRv#SHTJak#V2T*3mZ>0NkCwFX#ww~VQp<88&~>-fs#d4e
zLX-%u6bX>BP|%Z4XJm_lL4Ufnj)wG?qF8{Yi7IO~W?GFmwDRH{olLB{z%PwKPBz9K
z$4*cum!M8wuF4Y7XGxxwYq%Ex02k#sx&D}DS;o_Oc|k)%Q5rbUm!jf5@O?({i6;8d
zog__5*BhvWJY-b{>0wn4d0<r$QyQ`+BxcOM3&26EsrAd)PL+Wsp~>#x^4C##EMz-H
zh$SBgOe+3XJTULf=wQW&#O7NO-;em=l)sv>U*xY#uO8I9?cR;oX+FXNi}inZeiJDQ
zY+i3clUx3c&FQ#GCNf*K&R!&r+$c*@2c?QKUf{O<D2qBZRm>O|u3&z3m5SG>Ak2sh
zeav*auj10j%($ih`scrdnC-ju3_8>AEUqjr&D?(jJV`zRYoThHfD=2xm&{l|ZB6Cq
z7BjS@IM2o$&lzc;ck(=h+6LAr8`yD+G>4c6+|3T`PkE;i<c4gJqqjpo%8%I~A6tYl
zFS8qLv=kp4`Y#O%nAyh6g)xK$vlg1m;Zj&=h@dEQ&&0s?02?wpSU&#O;0(s+CHuM9
zz_9~n{sl7>2WP^f%%6v0T!-o07wUg7V*xYj)?g*f5BXS-$UYN*oinuhpYp-VU^O%(
z+s}Q<RSBZCGUOc*t_(_HNj(osL?_k5+`fe#H!$ZJ%()Vl#)9a4@EkL*Lih?;@p-r$
zX2a5TP(NA?SI2x*3W{SM+TpSU^#5Em#C3*x{twt-`N8vhEVLz_X&(a{Gp}Ig<#_+0
ziPfH?4RdQApT*2JxG~{$X($4#xi&WN_8b($4zoj2s<Yeu)gy1;+iV}S0&nxtVDr$e
z_sz}TeyhLQ>ek4nm+?^cI^{=$NT}UZSxBB!Wg)cNBV{QJc#P+NJyjLhR>SKA6`@{1
zw}>Av2q#pf<@vFy{62b2ROPh==_Gn))Fd6F$Y}Oof}z&r9=YXi1JstGAa2L^x*Z=B
zWMuHL=YCFIKbO?sT7ERx!k%DRBLjw^|ISfOb+F%SWm2p1^+$sn<vq8ft%qN(mE}RN
z<<((I;#ajs;Hq*@xjogbyS|T+{beGNmZll}e$KBQqIq+36B@i*S&ytnSG8+FB)WcN
z_G-cX$ZU4KPGq^qU;yk;Pk~X9)${86Ef>m<=_0cWyA>I}8$>zUBRmfmU42j!s8wtw
z%Jf}@88Z#^4;1+Uk<r4Kj5czxd_L_xGUY+L7p;8oz4t%3edliV;~#u<_oG`MeW=P9
zP&une)1&Ne4Gx{B6v0WfO8clz_t;f{@oqw$-SDsoOckKXt<r8{<RJ`lmi0*G?qOf*
zVU*ECJ61<C29$U}u#C2r>O8GvQU`TGR}TS$jyB1?1xZogMu*?Q<=;R-=Cvr+U}7%`
zix>DsnAj3G@pqB0@Jnb}N1aS>33Z!Vi~{;tsJ|@wXYXNiQ**LO%uc17<b=$DhP~LC
zg!SMfF$Rdl-i4vQgARJnk-6_agp{4;qG=QTee|68W*ZCZ&Uj_oUqn~Br>n2v0c+F6
zL%zW_CeG{>?V(!a97YDnSYTPSRIS2yA)8UH-bJSg85bAGnCUjh1)@cBf$Rv;(S-Z1
zX%O3L3vF1OjvNwszt(fVgbBpLd$=a6mljrsK-6b<pRP*23EA@|+qYuAbItN(%ZfO=
zxMes?^=>CXpa5synOXIvg;i(!rOv8_0R(Jx06J(3c@tI-h`k>2k&azp?{E}NxD^}h
zM7tFG!hb73G&AI%@E|j?LVhBe*`|%2Mv!UdqzH{Q%p%QaB1U(~PBg}Eu&4Y9d%~ZH
zhN1199pv@guq9-#hT5r($DHs6p?@Yc!;E%v?f3{j{|$cq^Iiwr_MvfZCzwY+vj%{f
zJ65vqr13tTordn0pRy<7iGlbG7EbcVcEC=wV?N|Dm{&wP104_XNzd@|WQO$tKLM{!
zz>`o@y20kmfU2NpVg?+s@4<b_KnGnFORqTe;*-=K%bV!y;+j~4YYS)B?)r#9y|#zg
z=QOK;wuw<8_*d|wNI1y^RSb0*H$?)emqz=KsDA3gTfgPP+8kAuBC`Xbjt~@Ry-;^(
z0qv<5-|>A{1u#svmFlV(!wuODVm;z_5ud0%zpDcEV+^WTnnovgL|_d29BAqt1Dki2
z6Ao1AB_bF+R{eSxi)vQBuv5Los4&aC9jxIK$S@!!P*kk&%c2Yf?Jug!?5I>*Nli$Y
zw7L*m0|edEAV1V140lnS!21p%zR=<spPM9!v`tGA`P^C-i;yIOJm}ZF1d(r>NKnL&
z=D;RBye9fgq-JyB@F<KU79zcIn$Un>WKk6Xo<0pU5VATp@s|Y3$!e=L;co)@w05*K
zGr5Nb9m6wzf%g}&cBW&>M`A+C$4~JPuthEc3Pb?l8FG|xC@&y_hC6EPo9bpDq)E{*
z*k;Dg<_Q~R$WcFGRST_FGb@b^7LNp6$-w@E#}0OoquB^*&==0OL}qEOZRvzg&7MX|
zbEihZrarY2p(35zzNO4CL5)}rQ_v>;Q#BN)pIsMkCTF)8DnY0Ng_!EJgNuE@E2#_D
z_OwSvz8KuJwqJ(lRi;=9Eh&nT$P(L0HD8SPfMGAjdm2Bq%c3{2V0qN5nCXCKBL<xk
z?VYlPG)nx6WSts@eg(1)r({(Mkwx(0O7$*U)gMyvO)9QXu}cM+g&4DbN;PsSHGId$
zH;FkZ`{1l1Gw2<<)u|ujz;VS<&8Drig4mPow=vGA6cQ8b02*66Xczbz{$7YlCu7ox
zl$3;Qj|QCgP&t7`-M7`NCwv5zfh9Fy!E}Z(6eQGRIBe-VShxmd9g%;WzSW5Ub6&hD
z=+&S!E8aBC31I0&jE#9<BF{7dCe1|v$NOl~UbXswYC0lMK`KPkL`vRR04efBtG#aP
z;c1ediV2BWN+b!M*1=0;gA*c+F%oBFi#HIB&yp*&BdsI`&CzU5X^vJ&Tc(HV0r5`R
z#$QagK~@&IHdxtAyldJ&Jbz5%D;M?Yc;qam%$dH@LXt~&|B&XUlV$?WyV1Y!Wo(+m
z&{Q-|FfbsOOFYJjksu7Ye>vbtNr=DzO5*twzQ+av$pWOIpR(hd)NkioPoZO@3|tvY
z=Nyeumc*(dV)qQ~_9z!8Bj`QcHMo6G*@JH)Be^sx=;R1ikXZ!UhP0nipq-l+E|TFC
zbtV>%0_T7a#oS3nc@LqbAJjT^gqs*PIk2m!FTxM`Oha4~!FqjxWBPbb6RQ4&$r^qV
zJiEs|m`NZ6MM9~}k<QJ@oX(J4N-|_H@yMt!NFS$AWt`;8D!gw!8eF^fXaL$%n1O<%
zwjkmXxp}MGXdudsp=YWsB-(%VTWl%?RD{^jA*^!+qvKg$L#?tL8MRhRX<XCfl<7Ub
z+b~6LknR~-2R&r=-8iIuotiaN>Ui5a#Cg>BA;km`4u!wkHLz&CqhyRmKcV&~Gsezk
z1CKw$rOgdP@LCiUu$tQSD#QKCi~ibar&bZ$cX4f3&cu{nqIthYg-r$d;IUE@CaWJ%
zLC&VSOU0j2p<$g=UfrjHD4CI;DZk5jaT%9Sz-J5F61;HLHj=-BZQ2F2WVJBX(Ix$v
zNlk|o(;P9rZVAVfmiA`wk>_#F$>@(d6whU&mG_XIxa%GV_mGF$c8~o2<y$qxFSUc9
z`=;z3bqEj2fBeHcx69M7Ed95)eE4v+z4Q~!dq3RXzWw3%w%?!m7PIw2YAX_MCq)ZV
zLZX?r?tcpfMVa`%0D}R)igLwq-3*Mt4B1dL5qQzZtI+U(5Wf_^fc9(CA+{sqK(#7X
zWaCz)97-(cZ(~+P3Zr^d&`|*$b*yd^FdkqBk+l!E1sJLD7EuE-D)5ac?wv6iH1e0Y
zVmySL6nKAQk#lJ>8RH+Ib0xbdD?ZJtiPBQ@b$Zk%PsL<aZKH_Zzrm#!v`I*X-oxo9
zPE)2Qp4TTWq=AzLuHwWQoxjZDj<l`J;SOmqp*fI%rKm932OaWueIiD5rz0ZF@&5?L
z^m`thT^!A?*k?McN?;CG7suwm9awk=ix<Ds(Kx{{j!YWQ=u-f4s94n8?pHp9Tv2+n
zk;otA@{+HM<s?KXH&nM>?!SUJZ!W&-bOSh*2OUYdQTno2uhs9n&QiN}Jc%$Wx%lig
zueFm!>CKq`ByTYU5vQY!mhDL{U!M@67A4NwUNAYCsglx^e~&B4=hN4vmwEr(EGJWG
zNYR_1oVa<51`^c4;vk(9e-xRdwahrJMaCDOMuV=WK1LN3kkn3vC5AHsn}5Ki4Gx3z
zHb=nJBb+0F$j~IvCP?R+Nk&a5T2<rLm#jyS_G?^|^-}O4(b=3$B9Y|Kr5B=XA%&$n
zktCYNSioXYW(IN-AXh`5gGSjgYtVNMsCrX0?GV8d@+<;5czpArug5w4SxkRhiwS*%
zX`>8qsD5AFL_IbUM5B{UBq-5#wMMVcQL#?1^T0d$qK+KE0!yQ8^6DSaFFD;cBr*|>
zugx(x=|+Uvi)yX~P8$Mk0|vD2|Do$gc>mdCIa?I7g!PJA=dhAy)$AmoB6bGwatIkm
znQ`K0LCM4~;Au&dTuwx*mdwPTBqZ4UD|~AC489V4Cj7So*p4H>w_P~m`1k>jYGD4k
zff2dTZ06B7ANPg&#$R-oralyj5c$7%aA_umAYe8rsksUvdIKF6Ic#5V)!MtVcH^J%
z^1pGh{^iuPRFR)o-tnxH6nDmC->1oF>3-=a>40)PV5k3$Z<K#Bg^)Mn=4<8ILo0v5
zcluYplIiNd^1>2j+3mPH<e%w`(sv@BuPnv+iV}#T9J(JwM&IkHI#K#M6&iNvC>2RT
zl)?9K{D%VZt&(vJXlU#mk+3-VDftoqc#&-@>6Q~7{S(F{iKDCV&{yEY6uICfq5cwY
z(tIl6I9*zF-a>_(axDbVI!XvZD^Zd#;0;{nroiDF$>Edp476U)gS*rJI7zs>^A)(H
zeugFC1ZqYQBE*|F9pguVii{rz$+r%EXK};%d$(>UTYZjki84~KTH@*{&010#(67;~
zkEtN{&^e<KA%KEzY6Mgysfq$VIG($8WzZvW1pkSHe;(PKZcAzUEP7w2qDO^3JmRQc
qbB4&&8btQ>_#XvskZ}6+6~#tSZNo{RV<{Lb7mB@=rImGi`F{bsAT7lJ

literal 0
HcmV?d00001

diff --git a/src/data/__pycache__/languages.cpython-37.pyc b/src/data/__pycache__/languages.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0526949658473080b788d97461e63a2066a57de7
GIT binary patch
literal 1048
zcmZuvOK;Oa5Oy3VPTYjP(g&rXf$|DM1aU-w=!>?{L{(FPWC>YrX5*M_*4D0_3NHMS
zN}RdyV>s@Wf5C|vyDb-1+ST{XeDlrDYPaik$H4Xd+pkZxl41NNa<!Hbyuq!0;sJ(h
zKmklBf(0cggAEm^f&(*9gZf3yEx-+!g*ljq1z3azEWt9Yz$)B?HMj-q(1hEt0WG)#
zcVRPFc1_eaQAw)#Ot*nmo574*gnJlSflauNnhkVlzyo*)&7kI5mzk@0GpM*FR%9kC
zF$*4DSZ*2GeWU%@Zxsx~ISX2^TH7fPd>Q(}87QPXU)f;ffRAw37okGT5brZNK?&o8
zC1YcU350th;G|(x+K=-BmQY3|ks#M$i0o}BGOwW|BALJ*qZIK$DlWT+RCSp6oQgP+
zqc9nzXi0oZ+)t)N1YKe`P0(XPx~|U?rkE2`{5Z-k6jS0jl1Un*l#>zilL%)Z2Q^04
z=h4(3JBq3ZGr=51efsANXeeEVY#?0%YNl<LPpj==K8`LAvr%+8$eu@6M_1!$JIqJX
zPJDd}U6GeJ(6-7S3qjtkUDGD`+7w)yOlWf$;+7a=+8m|Y3=?g}xMhTSrA;-}=2@W4
zfs%FnfN}v3%y2nNY$a!joFg_*$08k#EY8t~WBnwNH?ly?<r1aoPowo1H<zm!k=R>c
z&(E=293|&%OIO~V?0CJS)Aw!>ky}04`S3jF7QtTI*0$$CH1IrKBb|4&)!XlG9qTGp
z66w3^*U<ZTczTf4d;QbBZuhXa?>d=hrqs=U_&-uz;l2nw#gZR}d_ptO@#jTn6iryi
z7a*D6QPYs8aU6B<n}psl6@u|l1RW&@9q<#sLmsIgF!eP4qAM>a5u}{GrUy{;fR<S+
PEZENqYvy{PVOoX141GF;

literal 0
HcmV?d00001

diff --git a/src/data/__pycache__/supervised.cpython-37.pyc b/src/data/__pycache__/supervised.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3ac82ebd262b05b6e2b55c8d2ebda02cd42feced
GIT binary patch
literal 2286
zcmZuy&2Ah;5T5Ryota&)9Vf9NjsvVBL9zka5^w_yA%UZ1L4?9cXc%vgy<^WG-MzuK
z#upqUkjN86k?b2h058EI4=@KV;o=+IP%iLQuM@AM%xY??t9!evzWS>A)9LAsgXi(j
ze}1C_$9Y17XO9cz6TEsILOPOVPR>G>^N{Bv6uBF^jO;vX<zDDnU1WY3$cxgIt@|u&
zOHcauov<SV*~WJ&5wf$v<&^AVwkzl4d3gcav=m=D{rO{@<MiETelszV9#x6Dm+C}D
zd2Cd=)69_BX!UG7n-&|T!em+&(Po?$O=mDnBmLbdR!N^VV(r$=)d8nX@lZ5A-dlL}
z`w+%CWX9RzH6p8-<PU@t(m4|Q99B^~u(=QTK562TQ}aa!KDJhm8UEYnjVooj(YWzW
zsyF``KKbeQfB#zPGu5V9x8X(AxPvUM8ljDB+&qa3nAwh9pt@M4rXhZF)H!^fey}_&
z^JF<Lq)OsteJ{;MRaGvVM5ocls3?*wEjE|68Z67$V8L^6ma5&x-_wIqC7Qz19)!a#
zvl*t&Lpu|{fWgLp6-w=FA#%jc_JP3`JLEMxVm04liw>fujEGfbqdf^rAed3!use;A
zrJ-={Htg%LV>VK`5f#Wq!#=n1jKwTYX%<g?jgovlkrI*U$o_{AIB~ekE--Zo<L3^8
zj!p}&b-02;iQ0Cpv(OgENjQ7rwt&+v#$p^&P2*P0&Wt_d9=2+?Zf&vJn{yVOBew6u
z=GDFwN8IXu?US|k5x#pS{7txk=kJgELDSO181%e|v6>@sfdrLO<HtHyDn?dva3cp^
zA}crhftsf!uRw(E-F1+a-44at&Thlkc0#_kt6rsfBEtlDmtPKERplw2t?5{Y1+2qn
z?Qf2Cp{qAwO@EC6Cb6C#0+VzL=C@GB*$0R&01?}+x^2LPT0C?f@dL&j11JwMcf=(4
z|0BoB7Am2IFDk=+nIQ{1;QOBOYOm%}Jm6dYS@8DSI?gL%Z?Dh7^Hjc;ti)<6TmZ8G
zcbZnE(!z`(^c3o#%$q4doR5qxa7A^jh`<*0GkA7sSL`>S4V!7;9?5br%98>UuhH()
z;~gXGOA<~Ek}NxEBQM2$;pMnvJA(EE_6~rbVRMj_0My%?m^*$46?6O_9(0J(6n6%;
zATdLofzn`8@ED2#iwSD56wm`$fGEW_S2t==dq~L*!=-z`eqc4)DeNo8wj}`W7b~!N
zuzg|NlO*`X4cL6xhU3H-??7PX9IOGX^@-IdYu!@6m;k(Q%MO~HPwB&nf54}y+I#58
zZp|U5t(<w}TS~N%;&W&hK^+`+s7?;2CaPt2b?@q+%&U>1WS;`qJFWVqr6om8>X?$b
z_j*y9UJ7y~l&M7a(xPXEsqU>KTZ3Wf63x|hENytYiFP_n-<|A7d)Foo8Q&4TVXS*O
zdSOqd*i?fLx_5nVp^uGY+CAyTSyq0J!?H9_O>ghYQtxI}WxM1v?u+Y1L@|^Fap$9_
ztdD!7J8A28RSzDGN!$4<&PK^+N|nkI!!|HW14YHAsD0TktcV1S*ho#&Hi%Bs#no?5
z^n`Q7+ALmAvPf6)APFy{N5pD3(ngI2W~AaQs#LiSuyo_)i5Zp>H*Gm?{kU&#St5s`
z8pJPlk%brT)6;~!4Y%}L5NINtnn=I{w2<*TLtpOV4cHZ$<?21G#3g4h1y|7@)in~7
zq&xkY@FLw#OOxdoMH7|#9F%ZgZ)ZuYiY1v?ymd;oJBF!MjPh#Nwt_KubS;eWr3$l7
iOoV7#roC!{5A#xvvSfw)(=!m4ID!!!FYtoyO!t4x+%@w6

literal 0
HcmV?d00001

diff --git a/src/data/__pycache__/text_preprocessor.cpython-37.pyc b/src/data/__pycache__/text_preprocessor.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..81f26900ceddddc477c52e4ab6f33abf99150cdc
GIT binary patch
literal 1698
zcma)6OK%%D5GMDfl`J`rx@nyNg$_xOy+{Fas1Jsbq-k=fD-^KN7AlOzdPyrQ@5>|=
zrNRmUVgX4GIrJCQKKAeQ$ZJpe3qEy*YdI<E0_B3U!y!2we)A0<cRDQs?e|xI?mch_
z`3ncj24nINy4(iA2%{;<=#Wy>t<=iwp>1?KbuxG88r?~~tTAkW?lNydhW<0c8q8l1
z=JV#UJ#4Y&4hdQxz#|E$Zc0^@(?T!_gZ5y1_q%ZW(cszlkA8r02PZ;R9P&JwaWGuj
z$&2ZJl%_k1XBii<g>InXAEULQ%N__tD#|Fc-dpdeq9>NJE4ofj={xd<yrBmcvzfE-
zEAoUKQ;5eM?1KBo3oiBx$zuv0u%X)q^v|Hndmu)nB75CAW#j}L9hf5){##nnV~Va$
z>2KDN0}kyo+Wm(cjLx|Ea_Q#k9JakBhYjIW=iB}u!iVX=(pK-jrc;Q?l2c<ld)$9s
zWV|2cOz@~LUnJ?IEQ`M4G6@eRdCt=$AN8e(`z%rs*k7u!6ucBg%%v<uul)6_wV5VT
z#YI+bVb&$y&m*%3AJ>Jfh$k7(mF#V%MI5Da3zX%iPgDra85c<&SfUADv<(|UUqRh7
zDU&8{(FTj?fcW&pd;j;uz|q%`6`1hdb0P?VackGmMqtVTu;xYJ+DW5)bPl8mNL<Aj
zFc7_V*}<4%>46}$O;@S-9JCmR%Z6t>gf71V!O0NvKeT|>K%)g60+pBxRCkyMnfJvt
zSoAc;J=pM%{Tt|V14Km?CVfo-dn$SZSo4}zRz*UKp|q8Q<Xs0ljNAkm7kZ2!rv|gz
z+i<l>G~(SmvYW_mUZ}1_9tg!*H_5w*l|cFi4TFqs{TQVa{;d#&5O_uHK~huBgo^`v
zny1>0qxd-wZ0$&%9*8xVFiY$FAt7t*{4h-NM1|oMFiC`yP#+NU;9I4$oBzsFZ@J;3
zPVrFe28<Gv3d(j2FU!|ZYaxurRNPeth%vcAuu9)slwkE9)44SwbNl5+Wj!H#_vcRK
zz<LLu{2Bi4iB~zN0NEJqo20T&8nup9Ps!27Phc~Z{sG~c-Rd-(zg%#pdtN30;vKf9
z@ny1k&LEjzz~UM>w-iaP1QxPbM}d_Lm;p1j7nLQhcg0PdL4XYDw2RYHQ*{ZLn20dM
z10-Tj07Q>o2S^>-g6}$A`wvj-je~$b4O|Ub$)k*ip>Bm?R<KEmdOHk{CQ(|?h+7!X
zZ4@S(AB8DS<SQ1a#H;yMt7h;ZSkWtast$W`A<BuYi*6MkmmU@vvQX9>b8wjfMmNn4
mytE;hDJAU{`(qv}Qy7;;p#9A{@GVS@#D}5{yn@g6y}tpIFN-Sx

literal 0
HcmV?d00001

diff --git a/src/data/__pycache__/tsr_function__.cpython-37.pyc b/src/data/__pycache__/tsr_function__.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b695d10202be77c6549fb225bd65cd36e6716226
GIT binary patch
literal 9820
zcmb_iU5p%8R<3)is{f~FJTvyp*pA({<0PFpb~af8(i+>uStB7aM7v?xEt0Km_pKiH
zxVyT2tJ;|v)ri0ZJbnm*6+#FWG=mTV2?@a~JOL6CPiT3o2ZZ1O{D5{PBp$*8eCOQm
z>L1V88-*U#{ja*`|D1d7yfZgfvhe%M$N&7{_iL8*Z`2w6Y&34+iE_fSl%+(+@=Pr}
ziN5UEUB`0-joBTyoAYwryqD*`(<yX|Ua?#9O6a>P_lf0|RbCZPS5#4zP|vBds-Ui_
zIaNhHujbVP>IHQ|Euuc5meexpMRii0LcOF`)GF#_wWdy^KB>;Av#3w07t}e_E9$(e
zp<Y!N)J4>5>c`ZJ>hdSTJN?s^dP%+fiKSlNlirzNt|infs*ahn>Q(g`*1n*wsMk@S
zQ*Wzl>J7}CS6@<ZVx*?tQeQ@WL5Z(h^&fvi)LE~KRK#_e7T#;<MyC^W(!2^fjqN~T
zu+)sS-)+RYy+!Tju-znj(yxFghvzn)=qFGpk;qkR*G{ZNE?c`!A`=IFHxY@AelD?-
z9Qyg#N}R+^a!FpvPaQyGuW#UW{C;?|?xZsAr?M5|K^=#Hv3R=|#_u=oc7l2#wd1}{
z?G_5zZ>DxXZ0bB-)dY?%QbD5dr<Pto_1Vp9_j=vnS|e0CXk3dPwmXA<zjrMTqPE`}
zgkjKWha1---Mpq6apPJHlrh|l+r7~DulBdo;zkhreciho(FRckg=H7S=ua=A-*jjc
zKNk<RnIy|-096a<%RL+9uoGvGh%Z2Qey{Fo`ktPnqKd+^0YK_BgHDIRH^m9eq{T+R
z-`Vytzl3hYJBo_Xv`Z6$%|EJ%J-mMnjl?>z4yhD})_qGa6Z2!4SOkZ%0E--v-U1uO
z(4-6Pu+`JxJ`rN0(FPlM<5{rQ`n{+fw;u-n><g>dJ|g+B#DZ|7K802Ws6;0N^d)qN
zP<vum7*J742KZbe$^bJNfeX^v`XttmL9I&$ES2qzS>RRxTo`OT1=@LhfZ1t*j|(Cv
zH36FmHqn+i$yEF@8pJ^eu$|aS+_&|W1V+QuF(?ZT+h<@p4zUJQ9f9hMDXWPuPY7qI
zl>?&QOYKf?qYkUn_v5B#ql6^;5N8#HvsjdUzrEqN8qK(;U&c2kAYYdH97Zy-(mt6=
zeg_>QEAieC6s3J=?b$r)OpLlTx?)!6cs$P~uO5yUCdP;BjVu=#hLlk06BaFf7CTcP
z3Y_ozVJ`+|)D1`77Oy@=rkKCi@ChU2_8q|^Q%I7=Gkd;+#&eMbxIi6<Wk?$IAY-F(
z{)R!(tRAAiN=gE$Ge~NZ99^S=aMYKnc#(>WR9v9q5{hG#WqDRd{BBRZjrBuyzA7~7
z!W5>NAd-k4jj4gqVK&S_cnlzC`@MD;f7FfwzdMKrjn1Tse1!laVYLLyTLup4MJ8Q0
z&;jvaU3LWwzkp$viHOAkN+n4b63IPjdMnVM7_MOt&Y*0aA+k=GeZ7Xy)g43C(`f56
zRFI})lq??GM?H^L>V&XNvszLjBjsx{SU0gKQ<8$H30=oXh6dRkdRTsLp<&QK2}J?e
zY<swd3_$IH9MoRwD{MR+LP5zTLKmn`BLXlP5!c8FO9l%HiAQR0L=nsvWAg$JgoCH?
zXx?l4(St!l2Y0Y0;<yGl6fB3#{S}N&VUo$m8<<I~O@i0j6a*bZy{hD{jOBr?Y~bM#
zTh3T3h-D5XSdd#cwuubK&<9($hK(QI9C3xT&A8T#HA6g7(MUyemKRGp=mZZNVKaa+
z2p$ZAu(|!S_{I>KoX9acnn2EwArLZTh+c**Kmw9YlBD3bNZ!s^hh$@IfbSUaGk`}Z
zkT0v@6M78q(eBG`tJeEiICNAMu_*MH(4T^e<er74hK7Lw<%F=H$D%Sa7LL)v%UJ$A
zo1kCCA`j0<<EY<c?L&j;k!jpLbXXJ}T_vW5XJas0+3@FJ?5d@lJ*nJJrOL5zS}IRo
zYlXen3aGYgn~~T)KrEsPiM>kV4&O$zhk`>k#3(HijN<we@+=#F?w$MF^}nj#+IWk9
zKe(x1$1q|tsah-^Lz4f?&5Q|%Y6yO~f_^KtyNxa8Lh3{hbX<2+xfQ1Z(cR9Qspxyf
zMz`N-#{(723ejA%7q(!R;y^!abiR!*n&1mAJ15VIWpPnxGF6$|CY_#U6;Olltdf(4
zi2|}pkkl)BrVLOf^FU2b2#xK{b=%>4z36K}=ME$pCj;)HZfvawPX!!>DAqx^5#J+X
z3}p%thSb;5zmu_*tXKx=C+HZVg3hv7Q3Smy5inYt_O2T{h?eD^{6OyJ5Ia-kY;&)$
zTcj8ohPRXyk|GRjIVmM&)D_ef4pqLLR6-~Ym_HZ0$sEli6s;yzwhbRABKnvVDXET?
zi5a|c>7DOZ4!`&JU#aI(mm=V%353~~n|PZ;ZDIf;SD$CQ=FK-r?K^%eXv6~@M4BAA
z=gz$6xrogCzL%#85`YB=)6l01{WK3c`<V3dAheHBO$O7eVwhfmeIdi*6+v%A9i2fm
zD?MeFo{xg|#=W?YS0ggR6z)NKAU{=6l_rE=mK9l{`1BU$o3v>3BM#ld6a6}h!19E$
za9+Hmq_T0YvX!G;oTnU>Q+d>`DySmroGPg@>b$C`In)JJRr9Ee3g<!8CAFxQP?y!R
zI*GcXPN@~tb81zsp{}aa>I~|6bymHAdO@92=TV<fHFW{?qPhrPy{O9oifwn3j0F9t
zCdFe@Md(7cEg$wCC7dowoX1ewEc=LQ9K%TRFvhym2?rYLL`2>^4NuR_61jlLG;GJd
zzmK&;k4wfqCw3OIBiq%{qA8o20ZL&Fxh-;9uvHLzE)k6a&0~QS`lkrIg{<H+5=*Kg
zcKsI~YiE881j(>E8Ui|kjUh0*%JG>JA=D9j>7T?~zkx@;d&Ex7&$ClT<70NBNb9@V
zPD|4`v0{Cm-I%SUyGQS4w7vCP*?u!LY<-@+7&8{CBWXA`DXrhm_B#T<={?EJvxw8l
zm&`iF1RSoxBXc;GooSIykjyAeFC2|9hn&BYZ9g|AFV16A$YvXpz>GF#mp{3o;13?N
z*G0VfyV+hRCii-pk=cGfBBEcV4?hFZ8s@*3Z9awQF%*6dqV&OI5PcC-e~|6<6r#uM
zN4XRt?B=uh$pZWgH0g+TkZoO^q;cFonxQ|^zAVho*!ME_{KI4R4Zpy?>}Q{`?*;7n
zM@R45`Xcy}-abp5E@A$UkJ)#OFWGq;4<Qhd_>UB4c(9@X?Ht*DbCjacZ1P-4aTNbX
zicH)`n=LYh7Ben@(?WlO@!^TOB6cPO9ht+75&w^Xk%49hIs`10{_K&6Q+$7R99&3d
zk~bFgo#~>!l)}k|jc(xkY00O2!JtF!itpo$*fBHu9eR^UY^)sv#O8pGkmgWLw4XSY
zaw3C7c6>k?r_?5h${v=SD+e6EhDKCFQIMrvK_2_(@ailrO7FXPSP1?bjhMth{vIOn
z-?#UKK%_zknKk5$V@XMZMC?h5ayK2szwTx(amzT~(Bb9)nHyx1U9R(SVb9tv#6^@v
zE=ydN<5H4CHnD;_k9tlyyVb1^;&M_%`avQQ`j2rXsbFRv^&IMjWIid}&+ETQ3I}t^
z{2{?qJg5#^CA1cJtc+HbTUCV+C2>w!yVVB|f58Hj^GPvYP`OVfw@dK}(>|C_%7>yw
z$JPA$PURO7Tm_t8-l*;5YdGC>qYpXeX4d0m<utb15o%Y(+x?)vn7Vz9V^Hcw_Zs~m
ztx_mMdFo+m{v`FIK_AB?<a5+$SiGOg5P@_<Z$v#Er}<W=5ywH8+EEavPNyBAY-4+*
zA~EWO0nTN1L#Z@}Omh$_yc#vMZfvJE@80V48gczZD!!SDZ>8dXTA56iW3$_-ymQ+-
zaW{zKEd2M|YRfCpLY(0SLF6qP^Z{o(RKZpd`2fdTm>u=1*$BVr&9!u|>o*&5u+h`o
zURn1Bq4JTIZ--vleA{PiT%Pd?L|ES4w6J7CsySLbIq?lZG33f$Ks=)>C@5`RLOx*G
z92q#Qr6YqQs(g$nVAlLC2;IW^>>x@uDu;)~ox;Ny@ns@zQG(k>8y;?z+Kw6Lwu>+w
zXK#p<yMo!>TudQ*jxI`+;7byDT)-~%H?aq6Ts`lN6Y{Fp9CU*)9#Ps`9UjX3*%>hJ
zUAkBCJ>08M5u%8ZO`D0Y&yPOPY23vbuZde3+@4T&$B*fD<{tpZFvn9tvZ?|K>Nduv
zjwlWWY+6W1lt<Ph@kl<hTaH3f?UMp1f;*CTK^^u3Q`VYDO1U#OKpP<GhpW3Hw)Vxo
zwPx)j34+qzvwu@Qz?~9<j^Kp0N77wheIpeu*jVxz>D)+;Ncf_(GD3@bnp}=SoTTKl
z(sfdmX0O}7@lWV;nkHxMT*^Mdm`7KycY4i6C%S>gXg%q$Zh%0@h;-?z9V*ys(I%k!
z9RsET%S`h^hJl3?=V>GIhQiVJM=&5c`vC1H|BGKHc|;;Iv`k-eH~V7K`anK*9=rQC
zwRUk2wU6u%ZZF=5krbhMnrX>Xp!u(W2RPhXk8`oy7cmU5{+Za>7klEd0H1O#$%o!T
z?C!gUOOL@j9?#)0jPVio9$Rq^2}oR&ek#FZ`N(<Xg2!}WI<X2m;KuobLgG?JuU=l?
z`PTZNdpFRvUaO{iA4RoxSZnqm#~NvsW>7OGuf~S3b~&oyTCc{k_|!~0sv)Ar%V7j|
zKnt?u3psw0!LQ?C86+*l)vxud5TTY_;}nd7FtH3ulVOBs|9PxW^I<RC33Lx4g%l89
zjoQuLQ>LXnTt@Qsue1+sYKfg!kF?ce*_n84O7w}REc$eb#-d-{1@HFRQBDZF5htlP
zu}t4b0XP>Y2sm^O`XSAaU3-Y`zvCGRcl|kFjz#>7LrLlYC@aRK7@8gh-+zqZON;S6
z+~z9S9^ymjZ{l_~&EIV_@566Rft1HvZ_%3+{=j0KCW{e6pbhc%B`k*f4NF)L@Sgw>
zK_nccCh1`1LYpbcFgmmP*{5Y05kD(lhb0WQ`ufHb+R%vE&MVI&<>)2cFs<XgwDb*f
zlwZ}lr_)k9YKKuw88uBJph>@Vj|$Ro{WTP+%TBtU(;v~$78N@vJUe{%P2HpwWx7uu
zv7P{~=M66W1{)lHjjax2#qvS+N54lOz!R;au*l&dZ{{GSS+GlRgRA&^MewRj!A1t2
zjnGxRwT}ZGOpk;SVRPTIhsGYUm@y-8i@2dkY&HY-dd5NuhBMiJ2_2UGbB~SRu<;=T
zlH@h^haWP!P?y;khz*(JAdkaeScdk%!jb-v#_x$|>X2=0_Q!ZTJwdTd)a+?oZ4BW4
zdc*5lCKl<@5Acli=yhT~y<o(8tmk6|nsOJ+S~x8dI^1A<j7l><ewhY@0prM>E<S%J
zI6o3FBR(WK`j_adj@gEGU}-VY+E5Q!SY`py48hOgh5y7e%yO5+X`yK>6N3sm#Jk(H
z=<iVQUAZ_M-W8$;dwa+c5n2*0#6u7Q@Pv}nYq*hscQ7M-mBk}DGlDt%8wu_)CRala
zcx1e~h#1PH6uFX1lv8j_!rb(bC@*2Ykj)jd@p4jDHcqC0O$l?#DJ7-hl>u&?UI#9G
z^<e^b*QjB!7b2MLwC@Lqum@dS^9KzIwqsSVqz*+GCibwA%=R40Vxy$C+Z|8`6?&=c
z4!j&*q9Lj=reC8CDgSM(4KamzkS?{_(Y;_Kj|hbudi#Gc8=XdBA;nGVx`-H-OIZ`E
zl=PPRF=p%H4kSub>!P`!ad8{j-!|Bkqh`CmeHFF`;!8S|=3rzy?YqYCkcy|3;hKI!
zN0atx5tICHDf}-V{I3<}YMTFVgS42hmHASZ?=$%xkgwhNHjHm|_)<aNqQ#u)Zc&SZ
zIX28}a9G?JmkGLziBq_44$C*lh(vFrpy-`q(gnLvF1Uq!fqr?k9F+8zL#tRQmQGb)
MtE|yvX{k{CKiGfH?EnA(

literal 0
HcmV?d00001

diff --git a/src/data/embeddings.py b/src/data/embeddings.py
new file mode 100644
index 0000000..7056d3b
--- /dev/null
+++ b/src/data/embeddings.py
@@ -0,0 +1,196 @@
+import os
+import pickle
+import numpy as np
+from torchtext.vocab import Vectors
+import torch
+from abc import ABC, abstractmethod
+from data.supervised import get_supervised_embeddings
+
+
+class PretrainedEmbeddings(ABC):
+
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def vocabulary(self): pass
+
+    @abstractmethod
+    def dim(self): pass
+
+    @classmethod
+    def reindex(cls, words, word2index):
+        source_idx, target_idx = [], []
+        for i, word in enumerate(words):
+            if word not in word2index: continue
+            j = word2index[word]
+            source_idx.append(i)
+            target_idx.append(j)
+        source_idx = np.asarray(source_idx)
+        target_idx = np.asarray(target_idx)
+        return source_idx, target_idx
+
+
+class WordEmbeddings:
+
+    def __init__(self, lang, we, worddim):
+        self.lang = lang
+        self.we = we
+        self.worddim = worddim
+        self.dimword = {v:k for k,v in self.worddim.items()}
+
+    @classmethod
+    def load(cls, basedir, lang, word_preprocessor=None, dopickle=True):
+        filename = 'wiki.multi.{}.vec'.format(lang)
+        we_path = os.path.join(basedir, filename)
+
+        if dopickle and os.path.exists(we_path + '.pkl'):
+            print('loading pkl in {}'.format(we_path + '.pkl'))
+            (worddim, we) = pickle.load(open(we_path + '.pkl', 'rb'))
+        else:
+            word_registry=set()
+            lines = open(we_path).readlines()
+            nwords, dims = [int(x) for x in lines[0].split()]
+            print('reading we of {} dimensions'.format(dims))
+            we = np.zeros((nwords, dims), dtype=float)
+            worddim = {}
+            index = 0
+            for i, line in enumerate(lines[1:]):
+                if (i + 1) % 100 == 0:
+                    print('\r{}/{}'.format(i + 1, len(lines)), end='')
+                word, *vals = line.split()
+                wordp = word_preprocessor(word) if word_preprocessor is not None else word
+                if wordp:
+                    wordp=wordp[0]
+                    if wordp in word_registry:
+                        print('warning: word <{}> generates a duplicate <{}> after preprocessing'.format(word,wordp))
+                    elif len(vals) == dims:
+                        worddim[wordp] = index
+                        we[index, :] = np.array(vals).astype(float)
+                        index+=1
+                # else:
+                #     print('warning: word <{}> generates an empty string after preprocessing'.format(word))
+            we = we[:index]
+            print('load {} words'.format(index))
+            if dopickle:
+                print('saving...')
+                pickle.dump((worddim, we), open(we_path + '.pkl', 'wb'), pickle.HIGHEST_PROTOCOL)
+
+        return WordEmbeddings(lang, we, worddim)
+
+    def vocabulary(self):
+        return set(self.worddim.keys())
+
+    def __getitem__(self, key):
+        return self.we[self.worddim[key]]
+
+    def dim(self):
+        return self.we.shape[1]
+
+    def __contains__(self, key):
+        return key in self.worddim
+
+    def most_similar(self, word_vect, k):
+        if word_vect.ndim == 1:
+            word_vect = word_vect.reshape(1,-1)
+        assert word_vect.shape[1] == self.dim(), 'inconsistent dimensions'
+
+        sim = np.dot(word_vect,self.we.T)
+        order = np.argsort(-1*sim, axis=1)[:,:k]
+
+        similar_words = [[self.dimword[order[vi,ki]] for ki in range(k)] for vi in range(word_vect.shape[0])]
+        sim_scores = sim[:,order]
+        return similar_words, sim_scores
+
+    def get_vectors(self, wordlist):
+        indexes = np.array([self.worddim[w] for w in wordlist])
+        return self.we[indexes]
+
+    def restrict(self, vocabulary):
+        # vocabulary is a set of terms to be kept
+        active_vocabulary = sorted([w for w in vocabulary if w in self.worddim])
+        lost = len(vocabulary)-len(active_vocabulary)
+        if lost>0: #some termr are missing, so it will be replaced by UNK
+            print('warning: missing {} terms for lang {}'.format(lost, self.lang))
+        self.we = self.get_vectors(active_vocabulary)
+        assert self.we.shape[0]==len(active_vocabulary)
+        self.dimword={i:w for i,w in enumerate(active_vocabulary)}
+        self.worddim={w:i for i,w in enumerate(active_vocabulary)}
+        return self
+
+    @classmethod
+    def load_poly(cls, basedir, langs, lang_vocabularies=None, word_preprocessor=None):
+        if lang_vocabularies is None:
+            return cls.merge([cls.load(basedir,lang, word_preprocessor) for lang in langs])
+        else:
+            # assert all([l in lang_vocabularies for l in langs]), 'missing vocabulary for some languages'
+            return cls.merge([cls.load(basedir, lang, word_preprocessor).restrict(lang_vocabularies[lang]) for lang in langs])
+
+    @classmethod
+    def merge(cls, we_list):
+        assert all([isinstance(we, WordEmbeddings) for we in we_list]), \
+            'instances of {} expected'.format(WordEmbeddings.__name__)
+
+        polywe = []
+        worddim={}
+        offset=0
+        for we in we_list:
+            polywe.append(we.we)
+            worddim.update({'{}::{}'.format(we.lang, w):d+offset for w,d in we.worddim.items()})
+            offset=len(worddim)
+        polywe = np.vstack(polywe)
+
+        return WordEmbeddings(lang='poly', we=polywe, worddim=worddim)
+
+
+class FastTextWikiNews(Vectors):
+
+    url_base = 'Cant auto-download MUSE embeddings'
+    path = '/storage/andrea/FUNNELING/embeddings/wiki.multi.{}.vec'
+
+    def __init__(self, cache, language="en", **kwargs):
+        url = self.url_base.format(language)
+        name = self.path.format(language)
+        super(FastTextWikiNews, self).__init__(name, cache=cache, url=url, **kwargs)
+
+
+class FastTextMUSE(PretrainedEmbeddings):
+
+    def __init__(self, path, lang, limit=None):
+        super().__init__()
+        print(f'Loading fastText pretrained vectors from {path}')
+        assert os.path.exists(path), print(f'pre-trained vectors not found in {path}')
+        self.embed = FastTextWikiNews(path, lang, max_vectors=limit)
+        # print('Done')
+
+    def vocabulary(self):
+        return set(self.embed.stoi.keys())
+
+    def dim(self):
+        return self.embed.dim
+
+    def extract(self, words):
+        source_idx, target_idx = PretrainedEmbeddings.reindex(words, self.embed.stoi)
+        extraction = torch.zeros((len(words), self.dim()))
+        extraction[source_idx] = self.embed.vectors[target_idx]
+        return extraction
+
+
+def embedding_matrix(path, voc, lang):
+    vocabulary = np.asarray(list(zip(*sorted(voc.items(), key=lambda x:x[1])))[0])
+
+    print('[embedding matrix]')
+    print(f'# [pretrained-matrix: FastTextMUSE {lang}]')
+    pretrained = FastTextMUSE(path, lang)
+    P = pretrained.extract(vocabulary).numpy()
+    del pretrained
+    print(f'[embedding matrix done] of shape={P.shape}\n')
+
+    return vocabulary, P
+
+
+def WCE_matrix(Xtr, Ytr, lang):
+    print('\n# [supervised-matrix]')
+    S = get_supervised_embeddings(Xtr[lang], Ytr[lang])
+    print(f'[embedding matrix done] of shape={S.shape}\n')
+    return S
diff --git a/src/data/languages.py b/src/data/languages.py
new file mode 100644
index 0000000..2d03d8e
--- /dev/null
+++ b/src/data/languages.py
@@ -0,0 +1,42 @@
+"""
+bg = Bulgarian
+cs = Czech
+da = Danish
+de = German
+el = Greek
+en = English
+es = Spanish
+et = Estonian
+fi = Finnish
+fr = French
+hu = Hungarian
+it = Italian
+lt = Lithuanian
+lv = Latvian
+nl = Dutch
+mt = Maltese
+pl = Polish
+pt = Portuguese
+ro = Romanian
+sk = Slovak
+sl = Slovene
+sv = Swedish
+"""
+
+NLTK_LANGMAP = {'da': 'danish', 'nl': 'dutch', 'en': 'english', 'fi': 'finnish', 'fr': 'french', 'de': 'german',
+                'hu': 'hungarian', 'it': 'italian', 'pt': 'portuguese', 'ro': 'romanian', 'es': 'spanish', 'sv': 'swedish'}
+
+
+#top 10 languages in wikipedia order by the number of articles
+#LANGS_10_MOST_WIKI = ['en','fr','sv','de','es','it','pt','nl','pl','ro']
+
+#all languages in JRC-acquis v3
+JRC_LANGS = ['bg','cs','da','de','el','en','es','et','fi','fr','hu','it','lt','lv','mt','nl','pl','pt','ro','sk','sl','sv']
+JRC_LANGS_WITH_NLTK_STEMMING = ['da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'pt', 'es', 'sv'] # 'romanian deleted for incompatibility issues'
+
+RCV2_LANGS = ['ru', 'de', 'fr', 'sv', 'no', 'da', 'pt', 'it', 'es', 'jp', 'htw', 'nl']
+RCV2_LANGS_WITH_NLTK_STEMMING = ['de', 'fr', 'sv', 'da', 'pt', 'it', 'es', 'nl']
+
+lang_set = {'JRC_NLTK':JRC_LANGS_WITH_NLTK_STEMMING,   'JRC':JRC_LANGS,
+            'RCV2_NLTK':RCV2_LANGS_WITH_NLTK_STEMMING, 'RCV2':RCV2_LANGS}
+
diff --git a/src/data/reader/__init__.py b/src/data/reader/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/data/reader/__pycache__/__init__.cpython-37.pyc b/src/data/reader/__pycache__/__init__.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..317ad21cc65d8e8dfc9b58702bf9cc5f9ecbbbfc
GIT binary patch
literal 151
zcmZ?b<>g`kg8B8AVnOs{5CH>>K!yVl7qb9~6oz01O-8?!3`HPe1o6v3KO;XkRX;H=
zr6@H~zql+jr?j9TU%w=^I5R%2G%qhTCo?Zyzqlw_KP9mwQ6H!%CACODK0Y%qvm`!V
Xub}c4hfQvNN@-529mwR*K+FIDBsnG&

literal 0
HcmV?d00001

diff --git a/src/data/reader/__pycache__/jrcacquis_reader.cpython-37.pyc b/src/data/reader/__pycache__/jrcacquis_reader.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..939ec96f59c02df7cbe9746be915e08075c04c81
GIT binary patch
literal 10949
zcmcgy%a0pJdhc#F$>w8bI5V8Lq@|@D@6eXWk+hb*TH7mG56iYP5+vE$?Omm}#cIy<
zsL7_Pi}DP&2_!q-R}d%K068Ss0|y8KB!5DXQx1E{B|v~2`Va&N62Lj+u*gZ5<o8uK
z$q_BD6CW}IR##V7*W-KszOVY7sj0kzzuSNRj}P8?O;P@pZiat3R4(BcoYoYDDNOa0
zmTIah)iqBuHF-^$DO^*Y-b$P4<XOhd$Y<G>p((1;%9(ljME44<qFGFOO_@{ZmG-7v
zGv-XIWR_a9=4|q8&YZ)uj5psZo8{JmxgguK-dt<ZTuk~bnM>$nc+0I7b49l0yw%p4
zxhCs*?^x@&d0f^D=ySq6p(^hxtjMN5QP|XhW`2!Lvzbp6v%<d3X4%{)s`(O|XJxdU
zWD9H&*HdhXE#vxiw!&6%U1w|T7_Kj~<Lm^k-(YXDlk61U++bg4>u9O6w^)_Ef|ggD
zSJ|tNw5J;1V5iv`JUi{2acZo_>Id3^YSvkOTQxV?rg@f~HP5kg=6UA?JHMis-*jGa
z&O2w>H$T<P*PPVg!XaqrY&hRUYu$N`y~Z@>9J}yo_6M5zy7@-(WYc+rz0ThFG>z78
znHQaJvu_RFbl!4aKTuJ=aiAVzR*ds)c2Tx}OSWGewV$S*$%v;NJpF3pFzv2Vefu+#
zp{g29cevXYR<qk~2-k0Kp-JEM-8QaM+m5i9Eo^JYZ#mUeG{yXn+n#STtLb`9l=IvG
z9b6tnd0C~_psGcwJ6Eqq`rQw1-Nh)G5AVEp$7!OLd;iWA>*nRHcd0ph#qYL-!>JnV
zbem1jLDlf=_O=x`qCv|W{z157F)yRyD5lC3h_=c!uq?$=*p42}Vere1$6YtDuKJB`
z%V~>-kqkgr8siFn!5b(-HB>Am)KIEn3Z)k6C{tlNsDznzK1?%pNeMIfDV82)QKp%K
zd9<xa%~vy#=CY{JvAOMej%VRUr-euQzGL%97tWrD@(o)!+dg+e5+l+B$7}Lg%pK`%
zyXEkCR5|S@(svvi13dKEK3_z`UnzVImuENXME1JfX56vs!4uc(b~=7tIDt#-YCE3W
z-mV9{Q70bN(TO>{zRMeSOoq6j*4d8?%W~VUu&gr}Hz=Y|)J*cfsP@)APrBOhjV+KC
zuhWt(E6Q6|%V%AW>P5?X+_k;<2|tc8<+6_y$Ea-v#cBKkVt}E_`-X}-{ZU&QKgpPv
zQQ507^{Vn<xeo>nV8cSNfu+C(=tf_8oVgE%5WBak=}5IB)rnL$QvFEnK2z6w8N1OH
z&N_({cRHTkaH<+#z_5|l*qK;ZRIobSZ&>mbl8GRLLQ(VTvdXKdU2mkQ0e=Pj$kNJH
z{u&jb46wdKb)X$8{ZyFRO)+&v>Fctlp_Z1l6lxi!v-BrQm>TFTbBMhisQqlH@p}S&
z1{sz;RClunYFlGQ-(b0c!SbX2g}DEr+Rw!{v7Zm~B&Y|Ps-Tu0)iOy<<9~^EW7M9D
zYa$nB@Ln2XZ-lup%ceigf{HT`=5{QIB$&@}h4y|IzhDl<HmLp}51y#I$}hFY`rVk-
zRaFuf3a`~RqZH(Qg2vDCZ<1YT_^r;{l6b`LjM6&^%?TWfC3;$W)AJj)7rc$iuzT-2
zwK_!Iy%9?_SVhL=suMKOx+ku9b`aDXR9^9!^ZiRfW5;PZZ=H0y+<)RXPCn?($^rI7
zofNAsbxh$UEa0o}lfR1BBegTJO-^cc4L>R4*k9$jshRua`<KLi2dkr{oP1F4G>8!N
zN7oUlPvS;^)IvSb_z5Tg#49)<AY7rk2*LR=?4Gniy~S&-j@Yj>UB_b;w~dPDRMrFN
zYTgapb|CC_!--OX;87Zd+llfY*<RPV#<|bIue5wK$~C#)lD2$eN6|F5U2w>4S`N+h
zE><?u4@2#}{6!3W{<UgWT0Fjt*T)8ruh8u(6=d$J`AEBVFG|ZHqbz(5_k9s%Y=MP8
zjIvF)&0uDwPDVQ2NvY#RHfc?&1#0#jX-(zekxoOH1_Vr6uDC`P8mI?tE;Fa;I=0Yp
z4-%5NCXMk0nt@DlT#z|~yL?V`-$Gk(0)?VZYk3%tyjq3<S=Eeme5Wh8YWzFsF=9Hc
zT*r^D!F3d1x_Y4g0*sHHTBwknfThrc_A{lavJ_mM5o&|fVe;e`>d#cP>#|+8K`_!=
zUq!S@!C@w2(Pk6-m7;y0)|sg8DXZ!8hGT&Y()YbccSoxjZj^joK;)B#r2GS|kFAB;
z&y`0S*F*J)Ns-f6Q*t%2w@fz(Cy+^RTg1U&%nZ4>?~WHYnwQo}1eWuXpvigXVgrU*
zGZVAIkR7E$3bKI-OT#^qp@Pgz&iuXc%uu|P=d?MwR;ZmLqO>cVRshVD#mi*XqO@GS
zS;VLq%#z!Rx2y4Y@E*TPg%q-x2_k<;Gf6O`8tR<7s`B^IGTJUVF8W=eZck7MNRj}!
zAsK=csUk)4rGgBQDE$CFYwtoxezpdmrG)y5vYUp_f=7Ck<r|PQ_!DY9#5?d(*)UDD
z#~b$*VK8-&i^oO@yc%QHkT9jT)%y+yT!Bpq>{f^1iPiQ;qH^(Kh3rk`NBs(DA`x0p
z5&m=cFa~-|Eey2naI&g=iVD&b$t75ZPRD7pC}lGSvqA7R%0Mbxwy0+K4fHdM$ufKn
zNMz2))i})J&f~5V2+4yXatopqOiGmMxRbIUy)?NQ`-$%|+SO1pzK_WQGGdBa0&8?&
zDpGqxoyI?K1bNVr+LJ_%z)l-Cr1oU%gE;{UPmCf4?Oi1Ge{im!0)dHjsa^d*y#!{4
zDPU3D@ncjD(iC(^W~C&1j)6T{rV4WZRDwCn3i%D{m9PhP5}4azm34N634T{pK6b?p
zUB@<&9E;d-$hqL13bCb{<C|ExbR0dW4MFE0;67#!(E&Ku=YszZHQu6v7+BRMS0o}P
zZ-dcHW4IDdQafRFqWPqo^>BY^r1y8xFCf577>#I6rk3a}DNQzNL}{Y($iUvjjZh%;
z+TOCHYzdG}1At5IYLafn`-(KO6i*y6vQQ0upc9P&)u0Xsh!L7Z$JE%&ez}3uCrr(A
z299tszl;LbW(tJ2#140Y9p6htQ37ZJ7a@K_3%qfJHcctf?Z*C#%rKNe+M`Ujg8;#i
z&X$uW=VVVfQLLJloZ)0dqa3XuHcOZ2euhRLk{?k_ymVD|j_p+WMN9b@?~Y=Oyjs#0
z!7cs+w0@31(qNG(`!fC{ZV?U;fj|tR{}0@eIF=+wx`ATie!kG{jJPr(mD1h(ly;Z+
z!U}>R1i*l|Ex-L+IKatMNb*O8BX4|;4bhZ*nlM#xlU}<-1=*R2)gb;lGS=^-!*fJ0
z6VWZSj=aQBRwdCl@I(?_9VrEj4U|F~D@E)zAn+=Jz*xmc8<A>H-YqeyQ69j|_DEWS
zc*zsEX~v>t!q~X{0WCOoX{5s?l|Mwwz0dv|f6vbC2+_G%ulpV0w(J^0+L|lsadclt
zuzDu0*3Q>9>kXfGx(Ed*K%+m>84B3sHouEahZ&|m$^<2WaFQ@|s06D*?JMnb<i<n_
zSo@Vw*+$D=McRkuelE-j6|Qho=qwfHV(k5R6Tzn%d?eDaC925mW(NjIW8jI*AV)Rn
zVDn7x7oKW+*J-S$+JiR{yrsh&!dPSz`bCjP5LP_U`cugWQyAf&MS*2wj1Ox=0UY(1
z7R7KnL>dfj1!j;P7gGbYu-u_Gn2BeU<K@tM27U6>2lJPns(XJeO5w}_bmgh~;8*=w
zF&oap!_0+92_ogC1}8(*A7!4ZoMxQCQ>{Nwqp^ZqZ4p8DELJ_WgmAN8PF7n!RPn~|
zVXo=09QzhAH<%Z=V>1V^r~QSnB+B8!f%>{a>u8tZ(-y=+IFH#Ev9_hXUyH?XkyiHL
zZ=j0H{gwT)Si;Qn;R<Z*9A;a+ue4WnMd*+6nC%}_*kJ5S>92lIX=~qA!d0~XCQPCH
z_i%N0S>7!SR>D={LE8wISP8Q)VI7!<%|6nR;(I?_W~C)%*tQ1G$aI<fdmp=*3+F*0
z*@MlOlU{4^olOn>%533N3b{*iT<q%cpTpM{*do>}-wBtRsc}C%Svo?U)o?-5XZb)A
zbGvJYDU6g47qCL??y(|AS_w;Gj^>iyx)9R(3y(wjY&9$-?HRTvSNt#H7)Btz<lA$Q
z?_;v{H{y5*3Fzm=33*)$^`|L5C%%Th@LosS)7UvZdqq_6EPbGX5BdHY5(CE{rO<vG
zR9XYx$jV-~#Y^aw9renGYtj#%*n%rWC`s`^l$O}2w>CsZ737JWO4}EefU;IxIIQ>9
zO<=mNy-lHN#SP?B8>Vwx!i*=j=duccWa@$r?aElyD^JdIN*eWa7=trCy(YGMy-UM%
zBp})7#WO+2X}C?dQK76D5K3jkM=)FgF0Ejx72v)q#s=<O4|-*U7B&Zh+#dF>)oMMR
zU&QF;JBSA5lOb5eGn3}GBGu|;YaO@o$a8vSN{CWF32p&vXnT>yAAUv){_GsBdSyz6
z+IVoWLfH_{0X=}h*4MZ37=Pcp#A~<!2c!UPd)5p4pWWPCU$0PF?|JyP!Q7x>1M155
z7XUeITZZv*!XT*Td(~WS<2J3I7VbaXbs9oar`Z)<?o@cUEg686LZp5Cuy>MZ@%*Gj
z%OSI{uY7Oy^L+?F`oi|OgT3)46%E?u$!Pj=5ICF?&N3yz$zn@r$<I>lJQalCBFzsX
zy<>}=DC6t_fd)~!^$02FIQ*9olh9t21s{p_k?tcni1Nr0al3)JJx-w!fqX;>G0PBQ
zfMs!Pk)?+@r;Tug6G);!Jw{4|VI!S&Tb-zM<GpupT)TVEx_#%?y<1mq-IVc{MEBBd
zNh~EHj&!-zWBffitjwLu#iuDT9GLU&cH8HUg=9OWt{|ok;~&wK5`h^*?cu+RHgops
zt@~RyZ(Y7>eR$`l_0HwH*CN&9Kcq)_Nmm3k$Q<Wr-!jh;6>Z+wai36lE_Te~(f4tZ
zGn8XW*&{_l@1}^91dU9CR!C3GLQJJNb28;w`=dl%U`~Oc<J%}L#fqPj15lJqOEI&g
zBe>%>+9zSlOcTld9SdFJH1$*T2?z}+sS*WHT3#VQU;sGMT@l$69qrR9)rwTgr*m3f
zqaa`oJ<DiWl|4%Gt+-u7dkOE9@s6Qx;C~vuis~Y+C0vWh*U)H&W@zz@aVRmZ@lUXt
z=Wr8=m;@9QsDC+bLX4~<Xh2AJDC2HIQiIH)H15L`-q*Iet%nG!{ALAm3Z5N_ye=Xp
zM(&Kz`xraM@ziZ<yGF$W6v&kPkm_TkI03un;?o^M_BQa@e&7cGh8Kc73OWNI<Ta=A
zpWyD<OG)k&X9ZU9$PWN)A*v0so*fa?IBPy@C%~-JedxIu6#5&~=(UcGOTEQ7jv)fS
z3y)Ao=or{t4s4c;d*<Bc*$bN&&TpRCtkd|z_Ai*PhpeUXkV8`uvNy;W(BCwEq|lNZ
z{{ah23DXcMQVCOZq3%+UpdYII0>B$WwgjZ42N{tK0p|#$+>PHt=-p2zZy>`F{0ZKm
ze26x5#1ur6dj#-87GDCe+yIW3pd2#yT0awJ27qQb3m|xq85Cu>sIy#{PUZz*!x=#~
z%<j(MP$|y}5=f+25vi~L!l+Yn)KZuo%mN(Di}LOQWmIq$AWuF+L!KBIp)pvbu|uPu
zYyU0wTOtt0Z(;<%I;?N$^CKi}<=th5^${>&b99;tc$r_8Im3P-EF|L=FfPNm^K#t3
zz_{h#I_?SsNZMVN;}&ri8;@H|#+?eMl5wXnZc(fPDghu7{Km*jVlB)8UQCVXth41K
zoH_>hO}qje6zSdL5-{ZuhpcY(N@ICU-0wG}8-wffhhD4;PY=(t9@Hu~6W8pD8sCA`
z^lsdSVZ<XOKL9ZZo=C4!tK8%J<O%@`hM+=%5|Y!(H3;}eek0DZOD_-TF#BN}655fD
zA4X5&h8oGN%6ng~Nxl4S_yc+wL(L<hHB_fM)a=#0mWL$0L8l_5Vw{v7=`PRfE#7rJ
z_-c9)x4;sya*m7@MW$q|WXMZ-X&Le^Qs#4@9OBptqJSjxN^*=-)t;&K*e7E?;osgz
z5~z(c1FT1klERG%8!iLkEOc)e2%o}{15lT+5>0<mEX;q7)%5O+*FRM4(IzSzm^^Ss
znJBxxU-4l@cqQ)2hdq&sYCi(U!7QxutM8rvA(oWvo_|X9KSGggoBuI2Ja10r#;3;{
zk20QPKXHQI&GEM4mf=!^;igE`l4aY$XhUTDN4rdz`TxEle7G^<WG&f<->!6g$mBlh
zN&?jzUCwc;?(H9e^yPKH!eG4HCX)56_qx<c64GQ(GI<ySOgo@tVzbL7XOr|n0yeov
z^gj_}JIau9pNb9@66#TMn^S^`2UL)xM_DMHM1EP*?OZ1am`%W6lx}zss3-%S^4y1J
z`dw~!cK93gDy1zWBj`$_5pc>x;M^XEbes|iCcaLPnic10uFIxG3ZranX@VG~Uc)oK
zPel(!b(;T(I-g23;RTLVVk9~>Hw(i~UG7DhKx(!*)9iX4!8a>5aFH==Ff(Lu92TkD
zW|45%Q2Wi<_+4bU$4za-HU7sAKhiYRxN+08Y&vo;rFV2cA^y?Kgr5H=wi=LeQgp&+
zMW8RjR6t#X-%vAv*pLhdQX`Z$jWR}BWq=MsZWDJ}QKGd)jsFT`(su_f8(_p5sdx&_
zgxIC`r^MmW^D-o5@T~xS6hdbk$@v-1$Oc-R$wCP)h!W2ZQ{>%{Ue~EjhO-l=X9PGt
zXGJMvzY9o*917x-;mK81{fz7`42Kj3%7YJ$$R*S#s3?ud*N^C2mHYq@yk|eew<Jw&
zZ_~#sFS4N{I?!~Z1y*n)%{+ZIffIVZOeIrRkSy6A8hTZR&p7D%->h!^$m(w3xF%lU
z8M(kC#|9gC1P?X*$R!Sd>BImZy=+rD@cE;ibeuz!iads~q=<mAH*kRtNJ^NcQ8F<Z
zdxC_~!V5`uik3~F8HceDuOh^&sPX~s-M@p>xc`KLKc(B#=-E464}=eDk3O-v{^8cv
zwVUs4y&Hc}gY1J9CmO9M=Ocq|w{G7105tqFjKN(L0O=IzNuB4Eq#p-!yid0j$I0jy
znf4d;q)xq)l;=RW>b42^j|qI5?j+HJ-b=C%knk6FsjAnjxvj0Ods~v#Thg&{h6zC4
zpVB&hMg_Uy>f7`sPaNx87AI-qWNvYExgWs31vN52u^Hynz5qm0!(or()UJ8>&Y~^u
zp^os6nxevyHvT&GF9pbq*t}g!axj4m<~f<~DAyIPS0jrp?Y9iiIVm;&Qxs+?Hr6$=
zkU<BZ@JQP%tum*mh5wQUnWlkj!~8=a^9NC(;p1~G`59Y~9DT$mc<~A1G`&4TMU@H}
z>5~nK<M{hjqf-hQ!pC9#Cv^K0D##vj!c$V5;zO^SxQz;uZ<Y-2qddON62m&hT(J+0
zW{za)DF+@ENtmu)zIXZd<$E{cWPw5b<<X}ONGhT!IWY8rBIzJ1#MVJlCCbTpsNI}7
z(oU_DQQ{cD$l5Vlv+-v95!c($0wwq%3bNT`sYz+bQYY3Mck=((bP?Y(EE^eP#+Wkl
pM$X9K5bmClqr`0fEFy7Z8p?(^9kIGWV`!`DqPDCqQ^bzz{{T@_xLyDN

literal 0
HcmV?d00001

diff --git a/src/data/reader/__pycache__/rcv_reader.cpython-37.pyc b/src/data/reader/__pycache__/rcv_reader.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..104401da6adb3aa35cb2989155db32c363f69e10
GIT binary patch
literal 6717
zcmchb&vP3`cE_h@2EbtOgFi%(BK5;CZA*eB0Tk`EcNN(sTcT~nGObXwC6BG09ifNh
zkOK_V(~v0az^O8Qb(QzNRRJ%l$~}kF-s3|ml}h%ohdHGxC+}A6AK+8IuLpoArJT)=
zN<hx**FUGHyWgi@_q&yft>O1qfBD<{ufCvZ|3;DVuL|WFo_LjM8rL`rwKj7YBV7;m
zwxMfGYnx6^r5ItpU2qC*%du4442x~sv6Y?+OYO2#9@VTk6||5KtL-Ugsy*#YtMmfO
z&p0!qGPBMs%2?sC_M9^}N;~e%t2Zw=3%ux@;I^~KOU@E6J12RCSNRm5-Zq_6d}f<D
zr}?aNh9Bc|{5YTI3;YCM<V*Y{KgCb;GyDZ!`*p>6;eCz2$d|v?`0|17)DU}#zl_++
z{)-2OvyAvF{4C;URs5xG&3l=z@N@k9*UWi^U*@m!3((H;Req6Qf_9F-#v8l|?L61-
zX!X~>r;)6)wBQ8Y_kz%0uj}dQjq9JTy0_k0fB%kq_vYpY?)t6G58XSPYae}dbN&6)
zewvMnb{NFc-9ptEvC6hDUG7QG-HF;h+A?>epp)kPeUzdCCwcU^6Gk3aRor09?L^X@
zWEZYSy^i!ne{V<1?v-Zq@#Du0FKDy_+3<O<*%i^Q-;!~2w;hUR*X#NsZa#{_Ufcgk
zGxQ$^@yhF7(_2}+6p3K_l8m}RE55W7_`(ydohMDvdUUywr8X$FLB;wX{;ytkc@^*3
z^E;Q+AoQ;sX^eZ10?t*x8m*|^6n-4_M9Ytx(vM{;3PXHM5OreJv1Y$9sw;)xYIHDU
zjqT`Blg1NOdl(lK%tDlGhy54ACn8*3S$W-6J&0Xu*6qrb2I@4n`#)OX@}DZO`lkxK
zHZE}LKlDL(L1&!1KqG$X(du~69|}K=(KE-|So>siZDnO;qP;WG-koTdC)(;ndu^h<
zKGA+M(cYM7Z%(wIPPCtmw2eD%y^`vpm+IV4^(~R=@uO7lM2LH--j%5y;PGR}2t6pf
z-PG8Tk5M9Qk;ROE29#@f;&}*Pa~Rhgo#(j0P1u;BY%HIaVKMHFsD)M9k*T)c%A+*>
zDAe&>#}h9@BrMTfEioZkl7rL}9nwe)NRw+YS+-u!p?5qrF~@#thF)hoHKo5VQ@e$f
zZ7gZnVm>uvKim?FXjPmdahk*uiIWh2qY1!N`{CUtS!&bkaN&E+_)!q{y4{G@XW(x2
zIvqa@I@`@yw8qQO<G!Hf>Y_Z<M)yf-xo*%2r0c$o7Gp(gmLL7~k01T`#<*l%7bj47
zy>3$Nj@R~GH?>{29r0dBdf9a!_Pj955ig>w>hBRzqqGHxt9W7>MoU-mXJ!cdLN-HX
zD3cjt7)U)`T-)!$0DXQ7W7zYy{k8p;-z9@wuN#8qL%c-dWfC-6M?{UHuRy$qCwqGq
z|1ZCN`sR%dy!XHT(zAHs`$t|%@s<HWKl;i7p6htxSMl*u8)8izvVoB>aRuwk*wu%|
zp|)3`7-xG%yvp^XsnX=e0sBgo<mRF_FcY0?dqp7-<`y-XOUwgiYKFGQM2NKfBD7d^
zbf(GtSBw|9^|h85!vZfJvK!g~8|0{cZlkR{(k)&(WV^*dLFr|sTf8!~u|bY&PGvRG
z?_yGn|4EkkwCYm<J+VZ}XHeTBJ^p=SC564b_#@I(D+zLv;<j<fhUG(TV9QEk537gN
zkCL1kPA7OfdO8CuvteU-jKb_UIV^}{>tYVW25a~F=X@a|aizvbR$Gg<YF>>8TU)+>
z5!cA-XvH}BK>F=~3tDI2lW38kAFW6e&mBC958GOz-LDQ<!gk@GENZ{fADU#)so9FS
zk1+2^DS`)BnR;U-HDJxpEw~@-tF+x>E9t&{kD8@-<hA~C*Aubtjyv198b&QIjNgVb
zF5Yi8n$5N^J++(A3**nPwL&kB-&%eUbgyKa;+4ydmF4?Q%2uf8Tc{-i8T$Pav>li{
zc;KhZe&#c2^{LOCg`z&w&OS5yMD-PQrZ5D<zkot1>pu5w;u4a7j%WN)MS3-`*6jAH
zYwfOlGMNm8j(YA#*;b!T^|j5^-V#we#)x*d#jEtr{39>y`LQzF)Cf9KknN;}t)Rob
zFicHSGFXxgrS_*(evMW_nxlrW3xs|LdN;)OkncQ>1dlOo-L8*{#G-oHDebpIw-v#S
z0qjc{U|&dAQM!uv2i(QKAj3~h5k+VQ(}uLPO5sz2I~K)>@lKI6H49Fe)X6lcA`c?|
zM0GkX(Qvuw;kMj)ZgkV?vrFU_`Wml7Xv}6NtFn1!={B?2GBfo&t3og1Z%H?iZfPc)
z#xtjvbz2w2mejz2wP^X#Zw3#MM^WdWVE~FO`FL8w4JW(Z)&{y{2im~k#=zv}Ajflq
zJkP_5!HYdzT;tZjf-M#nv_Y}6gw!HZH@HoyNC7jT6$VAPxcawje??+PJOEw3W%rk1
z<K{5O%ZYuc$6r3&mH9+V>;oP4UPwv@EdE>c0~A=$cy$3rh;Rze^nxZWgfk0h=_@9S
zRCZ9_|1K#H?L+N`cK<hn3iMJ^8L_1LWCb*-f*KZ>VGO1~uxc{33tBNP(ZQBXGx@PM
z!Ogo`XVU~}_Uvz&c!S|60`AQ6x#6+I=Eu>K3ZC2{Mu=&jX`S;ZJ138C>8kt}OdH_X
z(MKydQb7N5`W}_i?@=4`)P|f-Q1?)$!^1bAFNj8?_T6pm;dgh(v0cP=@1o8@_V&f$
zF}W~2A@xIsaB&najlz=@9)j=mJ@Qn7onVU&3QO1#)0`yNj+_k`!wt-LzpgBy7Vkv8
zkk=mgwPjUg`63h)LKunKGM37+0wg@DPSAEVE%=u|lr+^uT@U_fA%XwAtcf+WIZimc
zrSx+MRA%4$f?xQe(GR-)+GkM@FzB3<wb=LDH5t`*yhnab?)Wuw!fSoA7IbR;3m0qf
zYg96vkju4JB)Yx0*77<voI0qx<I{-G`xk1Dn-Lhl6^XVN)^M_D2XPEvpi#RW`d;i`
ztO+0Vr?>UzsaUNY2_G*~HOOmd0B+=rh_^_brvduNF2;L7w+jYM@KyWep!PVBJE#gw
z#31z19R*v<13$b{^GCkPWUE27A6T=$Q2G3qgZt=<$EhLkK)IE121S+{KvDlBhT{ry
zW2p+YF&c=qy?q4r(`oe`MJ04OgPRfiQe2@<=OEx0`)RK2$<~e_4hqbVmR4v8-FM%)
zv*v!XaZ8Ztrxv+qw8f-m1Ox(T=vBste6LIF%Dm1$qgLLa8o4fT4CZBHrueIHEPhJ4
zdZ(M3ZQtvp26~<v=r@G74=)!lsn4WE+P&Pa2fqD`DinnO5JX3VYSdz<N<Eltgt2mU
zomsUgF$GUEoT@r1O-9ctsg*nevo#+4)C4v`ap?&<+0x=l3s{1g0?;ePhB5}efhJSt
zKaXE~%g1oKG=?2?Jl=%R^l9KV5B!=4P2GY<2u2#k4U^5XvZf==)|Zi9#h;F+jFd8)
zW~Tt{S$2Y1T3$VQc998~i#FjhU5`}oC@%k_;&MDIHOYp$@`qUG3^>XT*wB>5IQ5Xl
zD>BECI^woL+y-~^;O<6}G53Rk3AUo81Rf$to8$U`^3O{v$&I+YG$^1<E-4`20$nYT
z_^;8jL6qkvu@50Vd61asjC4if5|tek_kWiZ$%g}p|6yQ52g5;P_;Q2NlZKp1N{Pv>
zZ}dSK9G*_fyE7vW&nECHL9je8g2Q=m_!nU7F~wEf6e$kRaeH_?>pw4{FVrUr(T~J@
zhRO4CVZ`KnU~=UMlPh0VCQP2=<>%XzCy3A3)yzYau78Np5C86N#^X8S@x%8(<~crv
znv26FnFo^>hbKqjsZn^E!U>b%S&x`Jz22W6`=PNS8q7Ry!4eg0|A&}d{eQqD%P{$W
zKuGZcmV5ugk1+BgQIN<z9yY{B+!Qyd;(yA`GXgtT2Ey!m%)N1QBQ^K@C$adDax82Z
zy$8h7KSwkJssboc^(jSHpQWhym~wBE_zMzbtYU+NI<ZgKDagYX1divhDL$dJPf6S*
zK_>(88AQD*K1Yxi$DbREFDUOm3AzLzyvZG>O8LabSRLwxOZmSb;gNViqD6v}@JT#9
z4c8Ebt3Afs>Jagm_7V*ctLkS6airmofc7lR$j8iJc3#131u;OZjoK546%nBUR1m|(
z?bx|2OASsNlei!`fMEr2G56QWEriRd0~yzF<TZxop*G9`7$#6*EW)(uNULiDxSv3i
zj$`mW99Z-ucU8$nZJ1ABZZOTWcag$#Z;{Nur2%EiQ5!j|zX9m<VKKw#{+Ue2TW4|D
zWkZ`<T*R>(?fe=hULwfep`KfNa9@Raq;^4C0Ic;~8`d*z{Htoi8kUp7p=wE81r>2y
ztDGooe?E5n7(2IAe#iF)1}rt#qmJ*G8#raCVuH4wdL>((G_2{Y+SE5czJ2rh2RGL?
z)XuI};+x1-TdYv)Pc1!K<6?(|TAj)m?D3Jf<=2ZMq&D-}QPL?+u3c&S#-TAsdnt}f
z2<T{(YwZLfPYYQzcJeA7Kfkn&XG=>p>Q0*cd%P)bKxhnS60EFgtfX0Nk`|lA`2`yy
zw$n1!9Hr3;!<h!U=%YqmUth;*lpI*PO&e+TY_Ti9<~TO=2F%#^8*8CYw|1M*)0|35
zOGnbg3R=O@4Q@%}sg=>_3p)F!#hwhp26+jYYc(D3VXB0>AB9gTW09v~<~E#)v=GG&
z<=afWE<v*-=&+WSj$9WDx?@XoonE{9L|H%lB|u0Z)tR9wy}j}A^{13sr$%;FCSAv7
zx5Ubm!bR+CmPDR?c5{!k<0=h6pz~-vP8QqL4xMt-;^d|*^WM%<4jFw$FLFYZ5v%b&
zGNTF`HRPt&)$FS8Z5VXs4$=J4ES5226w>@*K99i_qCZ`S^P__$pvg$jQjSW?sM^zP
N+Awu{*<Q6@`FEz$(gy$l

literal 0
HcmV?d00001

diff --git a/src/data/reader/__pycache__/wikipedia_tools.cpython-37.pyc b/src/data/reader/__pycache__/wikipedia_tools.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cc4feef88e77683a68c11508513fcc40cbdf94bc
GIT binary patch
literal 10598
zcmcIqO>7)TcJA)!>FMeD;fRt*N}}a4Ey^1G4868CYiKE1*01fADXm1C+P0YSaH>f)
zIe)BbO5~&`o0#%?w1{Hwt`h_Sf+hD5<Oj%M5hRyjk=$|!a@bSQhYhmHML-bb5Cnk`
z#Q9#;%y4MSi_I=VsjI81tE#K3-uvG7Ud@F}#?kQi{crww{<puZY5z=x-d_Tl^SA`p
zHH~RZuWB{j)pg1nRl_ypJ?6&b-E>XdW7T-gaxK}$tR`x<Ys-ARnyfjl)9ssbQ)p{d
z)3uD7sb$@4ZO9#x&l1&KZI8RB+iut$MmxK@x3<sS*R9*{j<6(imJD~4rC9ob=8mxp
z%i?~34Y3^V2iYDrjQb(Bm+ixSoW00K*ysb@eTIFB9bgBMJA74RhuHW7jg5QHZpPd^
z%FnRFC?A&P=h#au&z?ie3HCBO!X}Y>o*iW`V7w#VBs;cZe5~_2E4W87!{gN#YRBAy
zu6f73!lv#Pyu;p6uYh@uu@jF>_k{Nh$|q#a$w%fL!#(Mq^2WVWUcs}xqwLhCA@9Xa
zgH1g$@ukKst#JA=?L<M3vQ6&SgVN$!y%PA1`V6Yf<%VC!J#oQ(>55<Vkj-3QSoSKx
zg*Cs*JYI-J$*M1c67#u;9GP82LlNz-l~+8fE%}S3dLt-#>u84oEMHXp3Jqm>qEc>p
zQTmOU(reeQm9D)ubM;n~qSDMaWT|4%-0+vhmBv-UB~C(60&PoY`nKN20<9kmj8Nay
zFKHj^^XWTAJJvRX7&AgMj0Gk=S&WTnn-HlW9%`YvX|&^k6~;I9wzVFiuSI?5{~kRP
zZ5usfp&cgB6ZQR`3G|JJ_NIYe$@MMtO3GfJpqJB5p_dt^LI=H2-|uCsUSVo8#^TIk
ziAS-I+;)2XW{?b=FwN9>nJ|NGPKU-;iY3Dga}H?R2HmH%Ag#(<84|;7yj|6Ynb2G`
z-qCbz^{;Q>n$zZ>FkjbhYPYr3&YZS-6W0jUVT&^}E&ERK*sIUo5rtO%25(e6A^iGM
zzFMv?t(BL&{Pd(amp?YirY2dTHIZ!YG!f0JFSmDhn<L53H=>Q-)hg1tuW30FuH?en
z;-UuuTY4PB9=42P{xP;uYgtF@3!?eeCwjh+jSTKZiAtl^#7I%ntFP5OUJkq{E=4nn
zRlPc=?%bk;kb|2@B9jD<Cn*~fYYUvvCQ5LxSuI1-vQaYd)`L>B9NdlK^v%MxStAfl
zJdN}!x}(muDJ4O6o&HCQ$Pmm;_RDTcWtEH@VzJkjDk5XikCLoWDOLTtC;nLD6PU-7
zt5bIyHE*h1XWT1KiF<x^t=Vi$1)lIp+3FrzFHH$vnPTOjJcUM3tf~9{ir<6+l}kaR
zQ5D7JMwBhl*h^lW7Nx<DU~oZ+raSrw{yDm>596Md>6mUHXBqp9F+Huz))iWD`XjZ9
z<GP4Tyn;ljQwT&L(QE4=I)wU`!D8F`7DRhnzoi8xgZfYlQl|HHDO%P{D--xZ)jK<J
zta#$p!UUnX#&bwo@wH&_)GMuHmpO0nykF0+*Q)u&2CtQaGkJUpcDEkn@8Wy&zQ|9C
zLYxnyUt|fd%q!TcCEi$TMi%ob4dzA8JLT${CkHDS{5fi=*W?nb<aw&e0%rnVt^_4H
zt23w*HWE$G>86&``ElecBsBC#caj>CDe)eX&|vxjC=D|{z;^-&034&T7OJS>>j54g
zz4%b;5d6eLeMH*^tbC{~>ZIHc?9jL*fkr;J>Di82%JoXl$OU@*Kc9a3X)7%i@a)8-
zm^kM|`u*14>DpQ~@Ub>|pLOd$w`qH6vLi^AFroQuTgS_^5LoyfW7WK+gbo@Pzt8=^
zD;SYce32BSPLOM0`AH}_E!Gofn)HVB?VOYZ5(`UfJzJ_k2m_DAtvFq6RLWIx4w+ut
zR%&{xH_vl;f!Sz6FHfO!!HiNtnJ;-TBT^xJ7R4|Me}>*kFD~f(Mas*~N)5c|Rn!W}
z67+G%E6Fhb71UK|Z|INYwZB)Npb)|U?Aj|s?iFf#dqt27z|FR?k6Ze-);4!`L-h_#
zmcVY?AH)u5D;B>^dlc%ZO|s+%Mt2;iJI>fcwbR50?9fy@y)$ALYLx<<zyx>iFVCQ=
z3x}NSnN$)!gxq5Y!>9Zta=+katHn2Xk%pZ@(p%>KUh^-Zb_SQ|Z*sYbZDU`w$xyME
z&^$^K9!WfddcKSr4g#mC4b4-7xP17u&E0vy6*Lrt2^uyxC*jG&CPN)bLHp{MLV-YC
z3gdqw6KKl-p%5S>MgURRMpCKue5k`-kF`x$cx@%Y#{o^)GzCkm<8!d{u<x-=SbkXg
z1eicv*?-t=s3;f#)O7y4u-q1yLXzx!I}s*`8%QzuvC4&s16tc&zZ=?UeW6VjI|l0x
zP#o3TPIqq3w$A?|faMP@FpC)M@wZ{Koe-m<43ZVbnfVp19bf%6Eb^Tovy}yKB(@E4
z2Q#zqR@iYhyVW~gd!3kR<*&gQ0TT%s!JY^`gS%dy`I4KJ`9>>0$?{SQ$$YGpt9?Wj
zz}U4qE4Iuc09QeA^4-@?x#d%>`4g?<vjo@5P6G`j1gTQ@Jv;9x3`IsmM5Z*HmLzZ@
zvU|jiAHxoD0wQIJ_fwASk;GVKr)hho8Y^Uv1Ntj>3#rKV>-S)4D*-=5Ee;}y?8ch1
z(NehFG;l+70fI@Pi{cW~qnOVGKTZ9lJW3hw3wm_8X8;BlhooGZ8*u~u#8D&~1kjN-
zGiRic->2I~8aW5p1N<?vxL`Lgq2+9az=!@wZ~GwhUodrK6iA-h@lGtzA&Z9tn2?U-
zoA2wyLObA-TUrPB+}r2C?2u$yp|us?27_ZTtUz#UTOWnHGOBe*z7lfe1WSx+7~Src
zY*|VM)^cLY4wI0*pN1y?QMU|(gM17eQkrP+-Q<=7*|eFnX>9A=65JXpErR?5=u#^N
z|0(*U!=#eJ-K}l`vK|`B_31$CHgy8Q5A_d?4`Yj_bcJl_LK;#IcZlr;-2PoV#rCz+
z8w=}~!W1-Tx}AYOWQc!*@$QE%q;Ve^(bzue-b~=_qhUJ8bii(Djo{84cx32c^Q#lk
z<rq7->**o&bWW=uHnkwT;($B<zK(Uo_r};bz2_TR-MCGqX9iYcsAI^rIJ{{D$>rQO
zbQRw^)X`9zWP`Oa_AHIE1MQ)Scl<}k=xDTR9pcpanbz2i20WP!(o#|#L9~ts9yITz
zt_%{#9u&j86kw6R?+17D&3^A=;68j0L@P@5IyUO1NXP$TYq%&!klIpg`jwTc*V^B$
zgBFnz^{U^zAs@56q^Al72stK2F@LR5W;C6J23~y(EEi5(9-I`<?|EDdK#T8*LJ?Ga
z;Y#bw;DApb?{nQZ@{M|RqqxhG5zo4XOAx>FiMBXx9e4eEJCsFPkgg5u^95PkV=MHn
zlCAtEjny?|uRc}Z)k2pRd81Ymeyv&cXiWy#On<EYM9HThJ(v9IC&+~0Rogv36O@>!
zZ`%lsjsdE09C{uLC2~tWhYml$2Z07^L>);}pyRRi!=N7KHp=tEWUo4e5i_tO=2>lZ
zW)8Z}641a9ej@Bu5GQ0<F~rBHw^1LddS<McJPKnSynR|*KaVjjIp%5DFtZKMw9~U)
zf2(p~{D8KV2%+cDI%5eOk!)KSpVn4?(6utjnJ0QuvX$1T&y)!$SIOLOKq#2E?y>&b
zsI|tT)51^7wV6LnZUVzMZy{*2t<zV7I(19T0G7#>08xh3?)s_-=0JB$y!p(ak3b}H
z5%1(LU}!K8;xVox{V}<bN(%ZI|0Fq0UeJ1zpL<9XqS={R>+N4i61u)_3BCy57!Z)&
z8x&0=YQiZ3QLv*pd`-WJ8@a~P3PvW0Q)I4q8$w#dC=Tvj6OpAjP{ER{41R2x2g-Iu
zmJ(6Lq%P3NCO=PaA&VI0`mmuK84RWHVn-pO%&w_#qMJC3M1ujd@h@#0lsN|uAdDMW
zR~lFA!Y{u{Ju0i|;F)9+cbvY*MB!VbnTv2Lsvf`)G$q+Zy-5~PC?E#b*&5s3VxWoc
zw2nD%)9wPH(4X`+k1hbOVzHPP<$DBU<@(0`@<!{B1SqPY4M5bY%mH5t^CIK^f@@xC
z!1385*+^u|B}4;7T_j-{#e5@E0J!;A=}Sns`B|#86xhOd0(43;EQ1Ydf9=cHuSi!E
zTfbIoM!DDDxcd6#TeGDbH?PlLzj*x`q7F9s^Au2sZ2E|L8PN<$cjhp>o8n%u#_J^l
zWOqnnl)vbEOnFWzDf~?i*jjfpY9^&_G1V>7C%f5hp=%2GZJK|c=AR^}lzSkBPYTYU
zjPZA<Da}QBV<-L(f>A{;L98CJ2Lqy}k07;W?19)*4p*K;P|q1Ac+~J-c;AReP=vyU
zP#%_U0P!n|TMQc%Ceohlt-|RP!Kre!T3i;5dU2ukQr|ZqKZ#_E+(B`!$Gl?TM!?ho
z<^`9Wf)lX)MhD0P<^>mhfO+95;iY0;%B7d=E8b^cU||^5B9^mgz&*4d*zE*Mf`ft4
zGl$}EU^#ydf54Kg%wiep!LkSqd;%_%*wny)l8Evx2ljFj+#<z>9+Ag!8)%(rXT#JM
z_)wVL2FC&SNG+!~b(RCd{}A2)N{ID=*@a+f{NKRca$rn07!%#o_^v%+I<zHEBEE$&
zR(~j2(y(M$10(F_FavDH0tWB<|KKnDGIocPT=1(%`r%vJG>XDV)B1bJbYL>bm@@4l
z$TG~E!OwII+AIc7oP2dq*)uY(LWb4u$hgWwy?zj_EwrAM&k}>rIy(QDVpfRZfF?70
zD~1i!;mJGbuhW=hl;t+3Mzt*y;Z&4BzcaTqmc>P;X=p}y7KHbaRV^=gRk6zo5|xpk
zK~O8H%m_tbm`2tQ;RswkS_4D2Lc_%GQQ%7H-vGY>6e92lThg~{{0%h!D_nvs8X_@g
zyRfiPB7Km^uw$Wiq5n?xbf4Ksd$nuweR@MtFgf`NGU96bAT{RC{18rB|CG<*wO+jh
z68RNBLRI%G_?yT-eH!C<BBNOQ6qkVSBUf=$Q?V?Vm#_84?$QEjFH!$?uU@^A!W(~s
z>z7)&euRtP!BE}!<?NM09R8|6Tv>Pl|2n;wVqMClSYDlilZBXC1R2{R*A%nuLp}0x
zh2)f<r>+Fje~C-%L!tpXP5q#<(SXa-x@oiuKljcn+uIE;^(}HMRaCT>Kvag7AZ|8-
z@L<C&PjRm=veMkeq_D*A(k-hT`Cmo;GggzAsE+i3lU#xFkdV5mg|hG~2)4tih~nNo
z;#D$8K++S%nvG^*h<ntlzd%v4FW&AxWduqv0y{#_m#FWs`l2o&lHq*0l5T=RodIHy
zc^a1XAWAH#11f%(syHQs(%XNvGJR@fe{n$1{uEsV@zAb?Gn9oJH?l(}?AwTLVbdtG
zj!30OR&7kTV@C{rA3ZA6t@j60rJctm(nvhb1!cyWCgSNJy~A($^|=OTVx|$?^vadH
z9-G0Dnq4XpStykvr&Ovn*jknH=~8JGaUS(Vs^LH)#cCaI_zAs=;KSB9m4EKJ?49F{
z`qHItoaBlO(_=Dx1B37uCBH_A1go7yzBj~!f5g>$FO?rNB>mtm!6_Mli@3y}Az@es
zI_NdvwbyWB2N;Hbg&LgQ)!@<S+D08-4h|f)b@FxEmXDao`a3f6Hv@ZUZ6)ZO4nS{?
zq7AGu;vsal$G?S$UUJ#NSsH9Qti6NigRv6lzd^OIur_=k#OU7FSEqvvrF3>yA2Z<{
ztvLLTx{(AtQBj02fvvJs*dAM!xLoz<+#+&rkc3MP*vb~CrVwFz_dKz-vlA_2VqW_5
zAc6Z4gDV0zJg?a8Ig30LgQ}v`Kpp>etPB4p66q&EdlCqw^F|}!*QnC*ab_oia=qdy
z4r<b=7QartXs0<Xdm*J5ED3m&toilQeUh)pxI8;Bz9~ojGIb(P8%z}kOA8IQ5hc1E
z2KJe@INDnx7F<$Zi99E3RQ?GML>`F-zal4(n@mu=VO%*Pe})Bumqm&H7TR3lUqM<S
zFPr{yxG0h)CE`6ONf_rRyD?Sl8}c#4bh~lez}%82w#=Z&6^^11tEC7U!_gFZ-DKFo
z(c*0zR}z=A{x6URb*}plU|lK6oUAaVqP?Nry(dFw(L%%tERUEJBD{!Ef&FDRUI8=9
zz@IfXV_<AHOA-r%H;;GCkF<v%M?)Yg=`c%oq7kcqBIW2E#H2EiqkmU&1olTglVObi
zD<at0Fhgx2Sw*xPlCi7tZc9hDM2s1+rm<aZ5z7MejDsQG#>un^S-y=k-`jIoGqk;f
zm2w7FDu=UcoNlue;#}tP9*T2G8BXJ<Zmlle1#*}s1%DmuzsoQanBT&+D^3Fg{e^Lw
zaWwvns7Ukw-!urJ5J=36iynf$rJV!KzH19Kdn#`s$ZTEA+)$^`(uc&s^iF{LWFCwP
zAq#n4eKNo3-B(h6c9N+Qsi8n=(2^;eSGYkTGW!1he0IjoH{@>{l>5l%F^j_RXS_l#
zoyPa@h?gm$GZGd4CR45qU5s)h@|@OE>Sknduhu}IjdYY(C?R|-<lG^ehtex$(WGhm
z5D#&b?7@A5Q!AoOt-QVy!;X3$AlU-f>Ba{UE_a7&^ldvW-66SB0~N!-1PKW^D6M!+
z8r~hgPtZ>o(65Vh<|g|C({VYOba<-q8V$yuvhmUQ&)R4@&JpL3&`B)$h2#gC^4~Dh
zJYd2=y^XsHTbNo_!ia@a+Z@iZEj%4D_;)cv->)FvL4ZOlF8&Y+o$A3mNE4S2KqiUx
z0~(CL2oQ#oJ_Evby>nVJ56nck?R*et_&EsNMud3P8Snw>hXbcjhuZxBu3l_8v5YW0
zfI}WE$xt7D&VhiqzJ8p}edj0Irt~;$dPCr-kQ@C-WK)LUV0J<T{K(7bo&xXEy*@V|
z!*@thoNMd!4dhmA?mpa=a@Hfnv=tzCR#mE)(#GdeCy)FNL6>@pc$6dFi+HG%I8mnH
zz<~PgkY9r4)@kgrIx)$YE4(2@|HmQ%3Znwvo>$8Ujr@X_hdM6RJ$41G@c?@FKa4hp
zKt*~?G*QL&fymetay-cC2#tZ0YdL?=4x(O>?(^^9PC+B#RX_7-`JGRL)f+`SMb3$A
zMn)Y!99XOSt7~4AeN70DlccMN4aN$fka961t8%x2pCd(al$ShCT8x9v@6l)<Q9|nI
zX0dKKjB0wkM9`vQc0_`(2eRq!P*Ym;X!!GsJVq0hOkcM(I`=##1^X~WeM0AdK)t~D
zYGoMf5<>gMsY{3(VUA*IuJ<Dltibr2H!rH6NR{T^n0>u8b8Yr(rCYO?-+JqfnX8z`
z98DoWf<2J=Zz1m<90Xg^Tv4KNs59EVM$HXQ>}<As=gVq{Hr-54H)XkJa3}yey10g@
z73e4@bmyy-d=&{cXhTGa27agllu*7=j9d@eeK###gCYV_wMJcf3;aFmlw1q^YLS9t
z1Hb4Yo){$&Kjre5FY;$S>i956TuPMJ>7t%O{m>_l-w8E0`0t`0mww6-%8`q!jvnP9
z<RvOnm`O79D0#VuemmSrVKxDb+!V=t-E7}Zl;~$XQI>*wMUn#g;gsOtr{;=n;jp3Q
z)yYX*qsl9koTp@mF3X&h6MI?_dkSS~f|?Qd5j4_28{QzLxD0y)|KfJScI<KIRp({E
Vq2cUzChW9hm|$48kwfdO{=ehW9QXhL

literal 0
HcmV?d00001

diff --git a/src/data/reader/jrcacquis_reader.py b/src/data/reader/jrcacquis_reader.py
new file mode 100644
index 0000000..c0441ed
--- /dev/null
+++ b/src/data/reader/jrcacquis_reader.py
@@ -0,0 +1,321 @@
+from __future__ import print_function
+import os, sys
+from os.path import join
+import tarfile
+import xml.etree.ElementTree as ET
+from sklearn.datasets import get_data_home
+import pickle
+from util.file import download_file, list_dirs, list_files
+import rdflib
+from rdflib.namespace import RDF, SKOS
+from rdflib import URIRef
+import zipfile
+from data.languages import JRC_LANGS
+from collections import Counter
+from random import shuffle
+from data.languages import lang_set
+
+"""
+JRC Acquis' Nomenclature:
+bg = Bulgarian
+cs = Czech
+da = Danish
+de = German
+el = Greek
+en = English
+es = Spanish
+et = Estonian
+fi = Finnish
+fr = French
+hu = Hungarian
+it = Italian
+lt = Lithuanian
+lv = Latvian
+nl = Dutch
+mt = Maltese
+pl = Polish
+pt = Portuguese
+ro = Romanian
+sk = Slovak
+sl = Slovene
+sv = Swedish
+"""
+
+class JRCAcquis_Document:
+    def __init__(self, id, name, lang, year, head, body, categories):
+        self.id = id
+        self.parallel_id = name
+        self.lang = lang
+        self.year = year
+        self.text = body if not head else head + "\n" + body
+        self.categories = categories
+
+# this is a workaround... for some reason, acutes are codified in a non-standard manner in titles
+# however, it seems that the title is often appearing as the first paragraph in the text/body (with
+# standard codification), so it might be preferable not to read the header after all (as here by default)
+def _proc_acute(text):
+    for ch in ['a','e','i','o','u']:
+        text = text.replace('%'+ch+'acute%',ch)
+    return text
+
+def parse_document(file, year, head=False):
+    root = ET.parse(file).getroot()
+
+    doc_name = root.attrib['n'] # e.g., '22006A0211(01)'
+    doc_lang = root.attrib['lang'] # e.g., 'es'
+    doc_id   = root.attrib['id'] # e.g., 'jrc22006A0211_01-es'
+    doc_categories = [cat.text for cat in root.findall('.//teiHeader/profileDesc/textClass/classCode[@scheme="eurovoc"]')]
+    doc_head = _proc_acute(root.find('.//text/body/head').text) if head else ''
+    doc_body = '\n'.join([p.text for p in root.findall('.//text/body/div[@type="body"]/p')])
+
+    def raise_if_empty(field, from_file):
+        if isinstance(field, str):
+            if not field.strip():
+                raise ValueError("Empty field in file %s" % from_file)
+
+    raise_if_empty(doc_name, file)
+    raise_if_empty(doc_lang, file)
+    raise_if_empty(doc_id, file)
+    if head: raise_if_empty(doc_head, file)
+    raise_if_empty(doc_body, file)
+
+    return JRCAcquis_Document(id=doc_id, name=doc_name, lang=doc_lang, year=year, head=doc_head, body=doc_body, categories=doc_categories)
+
+# removes documents without a counterpart in all other languages
+def _force_parallel(doclist, langs):
+    n_langs = len(langs)
+    par_id_count = Counter([d.parallel_id for d in doclist])
+    parallel_doc_ids = set([id for id,count in par_id_count.items() if count==n_langs])
+    return [doc for doc in doclist if doc.parallel_id in parallel_doc_ids]
+
+def random_sampling_avoiding_parallel(doclist):
+    random_order = list(range(len(doclist)))
+    shuffle(random_order)
+    sampled_request = []
+    parallel_ids = set()
+    for ind in random_order:
+        pid = doclist[ind].parallel_id
+        if pid not in parallel_ids:
+            sampled_request.append(doclist[ind])
+            parallel_ids.add(pid)
+    print('random_sampling_no_parallel:: from {} documents to {} documents'.format(len(doclist), len(sampled_request)))
+    return sampled_request
+
+
+#filters out documents which do not contain any category in the cat_filter list, and filter all labels not in cat_filter
+def _filter_by_category(doclist, cat_filter):
+    if not isinstance(cat_filter, frozenset):
+        cat_filter = frozenset(cat_filter)
+    filtered = []
+    for doc in doclist:
+        doc.categories = list(cat_filter & set(doc.categories))
+        if doc.categories:
+            doc.categories.sort()
+            filtered.append(doc)
+    print("filtered %d documents out without categories in the filter list" % (len(doclist) - len(filtered)))
+    return filtered
+
+#filters out categories with less than cat_threshold documents (and filters documents containing those categories)
+def _filter_by_frequency(doclist, cat_threshold):
+    cat_count = Counter()
+    for d in doclist:
+        cat_count.update(d.categories)
+
+    freq_categories = [cat for cat,count in cat_count.items() if count>cat_threshold]
+    freq_categories.sort()
+    return _filter_by_category(doclist, freq_categories), freq_categories
+
+#select top most_frequent categories (and filters documents containing those categories)
+def _most_common(doclist, most_frequent):
+    cat_count = Counter()
+    for d in doclist:
+        cat_count.update(d.categories)
+
+    freq_categories = [cat for cat,count in cat_count.most_common(most_frequent)]
+    freq_categories.sort()
+    return _filter_by_category(doclist, freq_categories), freq_categories
+
+def _get_categories(request):
+    final_cats = set()
+    for d in request:
+        final_cats.update(d.categories)
+    return list(final_cats)
+
+def fetch_jrcacquis(langs=None, data_path=None, years=None, ignore_unclassified=True, cat_filter=None, cat_threshold=0,
+                    parallel=None, most_frequent=-1, DOWNLOAD_URL_BASE ='http://optima.jrc.it/Acquis/JRC-Acquis.3.0/corpus/'):
+
+    assert parallel in [None, 'force', 'avoid'], 'parallel mode not supported'
+    if not langs:
+        langs = JRC_LANGS
+    else:
+        if isinstance(langs, str): langs = [langs]
+        for l in langs:
+            if l not in JRC_LANGS:
+                raise ValueError('Language %s is not among the valid languages in JRC-Acquis v3' % l)
+
+    if not data_path:
+        data_path = get_data_home()
+
+    if not os.path.exists(data_path):
+        os.mkdir(data_path)
+
+    request = []
+    total_read = 0
+    for l in langs:
+        file_name = 'jrc-'+l+'.tgz'
+        archive_path = join(data_path, file_name)
+
+        if not os.path.exists(archive_path):
+            print("downloading language-specific dataset (once and for all) into %s" % data_path)
+            DOWNLOAD_URL = join(DOWNLOAD_URL_BASE, file_name)
+            download_file(DOWNLOAD_URL, archive_path)
+            print("untarring dataset...")
+            tarfile.open(archive_path, 'r:gz').extractall(data_path)
+
+        documents_dir = join(data_path, l)
+
+        print("Reading documents...")
+        read = 0
+        for dir in list_dirs(documents_dir):
+            year = int(dir)
+            if years==None or year in years:
+                year_dir = join(documents_dir,dir)
+                pickle_name = join(data_path, 'jrc_' + l + '_' + dir + '.pickle')
+                if os.path.exists(pickle_name):
+                    print("loading from file %s" % pickle_name)
+                    l_y_documents = pickle.load(open(pickle_name, "rb"))
+                    read += len(l_y_documents)
+                else:
+                    l_y_documents = []
+                    all_documents = list_files(year_dir)
+                    empty = 0
+                    for i,doc_file in enumerate(all_documents):
+                        try:
+                            jrc_doc = parse_document(join(year_dir, doc_file), year)
+                        except ValueError:
+                            jrc_doc = None
+
+                        if jrc_doc and (not ignore_unclassified or jrc_doc.categories):
+                            l_y_documents.append(jrc_doc)
+                        else: empty += 1
+                        if len(all_documents)>50 and ((i+1) % (len(all_documents)/50) == 0):
+                            print('\r\tfrom %s: completed %d%%' % (year_dir, int((i+1)*100.0/len(all_documents))), end='')
+                        read+=1
+                    print('\r\tfrom %s: completed 100%% read %d documents (discarded %d without categories or empty fields)\n' % (year_dir, i+1, empty), end='')
+                    print("\t\t(Pickling object for future runs in %s)" % pickle_name)
+                    pickle.dump(l_y_documents, open(pickle_name, 'wb'), pickle.HIGHEST_PROTOCOL)
+                request += l_y_documents
+        print("Read %d documents for language %s\n" % (read, l))
+        total_read += read
+    print("Read %d documents in total" % (total_read))
+
+    if parallel=='force':
+        request = _force_parallel(request, langs)
+    elif parallel == 'avoid':
+        request = random_sampling_avoiding_parallel(request)
+
+    final_cats = _get_categories(request)
+
+    if cat_filter:
+        request = _filter_by_category(request, cat_filter)
+        final_cats = _get_categories(request)
+    if cat_threshold > 0:
+        request, final_cats = _filter_by_frequency(request, cat_threshold)
+    if most_frequent != -1 and len(final_cats) > most_frequent:
+        request, final_cats = _most_common(request, most_frequent)
+
+    return request, final_cats
+
+def print_cat_analysis(request):
+    cat_count = Counter()
+    for d in request:
+        cat_count.update(d.categories)
+    print("Number of active categories: {}".format(len(cat_count)))
+    print(cat_count.most_common())
+
+# inspects the Eurovoc thesaurus in order to select a subset of categories
+# currently, only 'broadest' policy (i.e., take all categories with no parent category), and 'all' is implemented
+def inspect_eurovoc(data_path, eurovoc_skos_core_concepts_filename='eurovoc_in_skos_core_concepts.rdf',
+                    eurovoc_url="http://publications.europa.eu/mdr/resource/thesaurus/eurovoc-20160630-0/skos/eurovoc_in_skos_core_concepts.zip",
+                    select="broadest"):
+
+    fullpath_pickle = join(data_path, select+'_concepts.pickle')
+    if os.path.exists(fullpath_pickle):
+        print("Pickled object found in %s. Loading it." % fullpath_pickle)
+        return pickle.load(open(fullpath_pickle,'rb'))
+
+    fullpath = join(data_path, eurovoc_skos_core_concepts_filename)
+    if not os.path.exists(fullpath):
+        print("Path %s does not exist. Trying to download the skos EuroVoc file from %s" % (data_path, eurovoc_url))
+        download_file(eurovoc_url, fullpath)
+        print("Unzipping file...")
+        zipped = zipfile.ZipFile(data_path + '.zip', 'r')
+        zipped.extract("eurovoc_in_skos_core_concepts.rdf", data_path)
+        zipped.close()
+
+    print("Parsing %s" %fullpath)
+    g = rdflib.Graph()
+    g.parse(location=fullpath, format="application/rdf+xml")
+
+    if select == "all":
+        print("Selecting all concepts")
+        all_concepts = list(g.subjects(RDF.type, SKOS.Concept))
+        all_concepts = [c.toPython().split('/')[-1] for c in all_concepts]
+        all_concepts.sort()
+        selected_concepts = all_concepts
+    elif select=="broadest":
+        print("Selecting broadest concepts (those without any other broader concept linked to it)")
+        all_concepts = set(g.subjects(RDF.type, SKOS.Concept))
+        narrower_concepts = set(g.subjects(SKOS.broader, None))
+        broadest_concepts = [c.toPython().split('/')[-1] for c in (all_concepts - narrower_concepts)]
+        broadest_concepts.sort()
+        selected_concepts = broadest_concepts
+    elif select=="leaves":
+        print("Selecting leaves concepts (those not linked as broader of any other concept)")
+        all_concepts = set(g.subjects(RDF.type, SKOS.Concept))
+        broad_concepts = set(g.objects(None, SKOS.broader))
+        leave_concepts = [c.toPython().split('/')[-1] for c in (all_concepts - broad_concepts)]
+        leave_concepts.sort()
+        selected_concepts = leave_concepts
+    else:
+        raise ValueError("Selection policy %s is not currently supported" % select)
+
+    print("%d %s concepts found" % (len(selected_concepts), leave_concepts))
+    print("Pickling concept list for faster further requests in %s" % fullpath_pickle)
+    pickle.dump(selected_concepts, open(fullpath_pickle, 'wb'), pickle.HIGHEST_PROTOCOL)
+
+    return selected_concepts
+
+if __name__ == '__main__':
+
+    def single_label_fragment(doclist):
+        single = [d for d in doclist if len(d.categories) < 2]
+        final_categories = set([d.categories[0] if d.categories else [] for d in single])
+        print('{} single-label documents ({} categories) from the original {} documents'.format(len(single),
+                                                                                                len(final_categories),
+                                                                                                len(doclist)))
+        return single, list(final_categories)
+
+    train_years = list(range(1986, 2006))
+    test_years = [2006]
+    cat_policy = 'leaves'
+    most_common_cat = 300
+    # JRC_DATAPATH = "/media/moreo/1TB Volume/Datasets/JRC_Acquis_v3"
+    JRC_DATAPATH = "/storage/andrea/FUNNELING/data/JRC_Acquis_v3"
+    langs = lang_set['JRC_NLTK']
+    cat_list = inspect_eurovoc(JRC_DATAPATH, select=cat_policy)
+    sys.exit()
+
+    training_docs, label_names = fetch_jrcacquis(langs=langs, data_path=JRC_DATAPATH, years=train_years,cat_filter=cat_list, cat_threshold=1, parallel=None,most_frequent=most_common_cat)
+    test_docs, label_namestest = fetch_jrcacquis(langs=langs, data_path=JRC_DATAPATH, years=test_years, cat_filter=label_names,parallel='force')
+
+    print('JRC-train: {} documents, {} labels'.format(len(training_docs), len(label_names)))
+    print('JRC-test: {} documents, {} labels'.format(len(test_docs), len(label_namestest)))
+
+    training_docs, label_names = single_label_fragment(training_docs)
+    test_docs, label_namestest = single_label_fragment(test_docs)
+
+    print('JRC-train: {} documents, {} labels'.format(len(training_docs), len(label_names)))
+    print('JRC-test: {} documents, {} labels'.format(len(test_docs), len(label_namestest)))
+
+
diff --git a/src/data/reader/rcv_reader.py b/src/data/reader/rcv_reader.py
new file mode 100644
index 0000000..cd4b416
--- /dev/null
+++ b/src/data/reader/rcv_reader.py
@@ -0,0 +1,225 @@
+from zipfile import ZipFile
+import xml.etree.ElementTree as ET
+from data.languages import RCV2_LANGS_WITH_NLTK_STEMMING, RCV2_LANGS
+from util.file import list_files
+from sklearn.datasets import get_data_home
+import gzip
+from os.path import join, exists
+from util.file import download_file_if_not_exists
+import re
+from collections import Counter
+import numpy as np
+import sys
+
+"""
+RCV2's Nomenclature:
+ru = Russian
+da = Danish
+de = German
+es = Spanish
+lat = Spanish Latin-American (actually is also 'es' in the collection)
+fr = French
+it = Italian
+nl = Dutch
+pt = Portuguese
+sv = Swedish
+ja = Japanese
+htw = Chinese
+no = Norwegian
+"""
+
+RCV1_TOPICHIER_URL = "http://www.ai.mit.edu/projects/jmlr/papers/volume5/lewis04a/a02-orig-topics-hierarchy/rcv1.topics.hier.orig"
+RCV1PROC_BASE_URL= 'http://www.ai.mit.edu/projects/jmlr/papers/volume5/lewis04a/a12-token-files'
+RCV1_BASE_URL = "http://www.daviddlewis.com/resources/testcollections/rcv1/"
+RCV2_BASE_URL = "http://trec.nist.gov/data/reuters/reuters.html"
+
+rcv1_test_data_gz = ['lyrl2004_tokens_test_pt0.dat.gz',
+             'lyrl2004_tokens_test_pt1.dat.gz',
+             'lyrl2004_tokens_test_pt2.dat.gz',
+             'lyrl2004_tokens_test_pt3.dat.gz']
+
+rcv1_train_data_gz = ['lyrl2004_tokens_train.dat.gz']
+
+rcv1_doc_cats_data_gz = 'rcv1-v2.topics.qrels.gz'
+
+RCV2_LANG_DIR = {'ru':'REUTE000',
+                 'de':'REUTE00A',
+                 'fr':'REUTE00B',
+                 'sv':'REUTE001',
+                 'no':'REUTE002',
+                 'da':'REUTE003',
+                 'pt':'REUTE004',
+                 'it':'REUTE005',
+                 'es':'REUTE006',
+                 'lat':'REUTE007',
+                 'jp':'REUTE008',
+                 'htw':'REUTE009',
+                 'nl':'REUTERS_'}
+
+
+class RCV_Document:
+
+    def __init__(self, id, text, categories, date='', lang=None):
+        self.id = id
+        self.date = date
+        self.lang = lang
+        self.text = text
+        self.categories = categories
+
+
+class ExpectedLanguageException(Exception): pass
+class IDRangeException(Exception): pass
+
+
+nwords = []
+
+def parse_document(xml_content, assert_lang=None, valid_id_range=None):
+    root = ET.fromstring(xml_content)
+    if assert_lang:
+        if assert_lang not in root.attrib.values():
+            if assert_lang != 'jp' or 'ja' not in root.attrib.values():  # some documents are attributed to 'ja', others to 'jp'
+                raise ExpectedLanguageException('error: document of a different language')
+
+    doc_id = root.attrib['itemid']
+    if valid_id_range is not None:
+        if not valid_id_range[0] <= int(doc_id) <= valid_id_range[1]:
+            raise IDRangeException
+
+    doc_categories = [cat.attrib['code'] for cat in
+                      root.findall('.//metadata/codes[@class="bip:topics:1.0"]/code')]
+
+    doc_date = root.attrib['date']
+    doc_title = root.find('.//title').text
+    doc_headline = root.find('.//headline').text
+    doc_body = '\n'.join([p.text for p in root.findall('.//text/p')])
+
+    if not doc_body:
+        raise ValueError('Empty document')
+
+    if doc_title is None: doc_title = ''
+    if doc_headline is None or doc_headline in doc_title: doc_headline = ''
+    text = '\n'.join([doc_title, doc_headline, doc_body]).strip()
+
+    text_length = len(text.split())
+    global nwords
+    nwords.append(text_length)
+
+    return RCV_Document(id=doc_id, text=text, categories=doc_categories, date=doc_date, lang=assert_lang)
+
+
+def fetch_RCV1(data_path, split='all'):
+
+    assert split in ['train', 'test', 'all'], 'split should be "train", "test", or "all"'
+
+    request = []
+    labels = set()
+    read_documents = 0
+    lang = 'en'
+
+    training_documents = 23149
+    test_documents = 781265
+
+    if split == 'all':
+        split_range = (2286, 810596)
+        expected = training_documents+test_documents
+    elif split == 'train':
+        split_range = (2286, 26150)
+        expected = training_documents
+    else:
+        split_range = (26151, 810596)
+        expected = test_documents
+
+    global nwords
+    nwords=[]
+    for part in list_files(data_path):
+        if not re.match('\d+\.zip', part): continue
+        target_file = join(data_path, part)
+        assert exists(target_file), \
+            "You don't seem to have the file "+part+" in " + data_path + ", and the RCV1 corpus can not be downloaded"+\
+            " w/o a formal permission. Please, refer to " + RCV1_BASE_URL + " for more information."
+        zipfile = ZipFile(target_file)
+        for xmlfile in zipfile.namelist():
+            xmlcontent = zipfile.open(xmlfile).read()
+            try:
+                doc = parse_document(xmlcontent, assert_lang=lang, valid_id_range=split_range)
+                labels.update(doc.categories)
+                request.append(doc)
+                read_documents += 1
+            except ValueError:
+                print('\n\tskipping document {} with inconsistent language label: expected language {}'.format(part+'/'+xmlfile, lang))
+            except (IDRangeException, ExpectedLanguageException) as e:
+                pass
+            print('\r[{}] read {} documents'.format(part, len(request)), end='')
+            if read_documents == expected: break
+        if read_documents == expected: break
+    print()
+    print('ave:{} std {} min {} max {}'.format(np.mean(nwords), np.std(nwords), np.min(nwords), np.max(nwords)))
+    return request, list(labels)
+
+
+def fetch_RCV2(data_path, languages=None):
+
+    if not languages:
+        languages = list(RCV2_LANG_DIR.keys())
+    else:
+        assert set(languages).issubset(set(RCV2_LANG_DIR.keys())), 'languages not in scope'
+
+    request = []
+    labels = set()
+    global nwords
+    nwords=[]
+    for lang in languages:
+        path = join(data_path, RCV2_LANG_DIR[lang])
+        lang_docs_read = 0
+        for part in list_files(path):
+            target_file = join(path, part)
+            assert exists(target_file), \
+                "You don't seem to have the file "+part+" in " + path + ", and the RCV2 corpus can not be downloaded"+\
+                " w/o a formal permission. Please, refer to " + RCV2_BASE_URL + " for more information."
+            zipfile = ZipFile(target_file)
+            for xmlfile in zipfile.namelist():
+                xmlcontent = zipfile.open(xmlfile).read()
+                try:
+                    doc = parse_document(xmlcontent, assert_lang=lang)
+                    labels.update(doc.categories)
+                    request.append(doc)
+                    lang_docs_read += 1
+                except ValueError:
+                    print('\n\tskipping document {} with inconsistent language label: expected language {}'.format(RCV2_LANG_DIR[lang]+'/'+part+'/'+xmlfile, lang))
+                except (IDRangeException, ExpectedLanguageException) as e:
+                    pass
+                print('\r[{}] read {} documents, {} for language {}'.format(RCV2_LANG_DIR[lang]+'/'+part, len(request), lang_docs_read, lang), end='')
+        print()
+    print('ave:{} std {} min {} max {}'.format(np.mean(nwords), np.std(nwords), np.min(nwords), np.max(nwords)))
+    return request, list(labels)
+
+
+def fetch_topic_hierarchy(path, topics='all'):
+    assert topics in ['all', 'leaves']
+
+    download_file_if_not_exists(RCV1_TOPICHIER_URL, path)
+    hierarchy = {}
+    for line in open(path, 'rt'):
+        parts = line.strip().split()
+        parent,child = parts[1],parts[3]
+        if parent not in hierarchy:
+            hierarchy[parent]=[]
+        hierarchy[parent].append(child)
+
+    del hierarchy['None']
+    del hierarchy['Root']
+    print(hierarchy)
+
+    if topics=='all':
+        topics = set(hierarchy.keys())
+        for parent in hierarchy.keys():
+            topics.update(hierarchy[parent])
+        return list(topics)
+    elif topics=='leaves':
+        parents = set(hierarchy.keys())
+        childs = set()
+        for parent in hierarchy.keys():
+            childs.update(hierarchy[parent])
+        return list(childs.difference(parents))
+
+
diff --git a/src/data/reader/wikipedia_tools.py b/src/data/reader/wikipedia_tools.py
new file mode 100644
index 0000000..f30816c
--- /dev/null
+++ b/src/data/reader/wikipedia_tools.py
@@ -0,0 +1,304 @@
+from __future__ import print_function
+import ijson
+import os, sys
+from os.path import join
+from bz2 import BZ2File
+from ijson.common import ObjectBuilder
+import pickle
+from util.file import list_dirs, list_files, makedirs_if_not_exist
+from itertools import islice
+import re
+from xml.sax.saxutils import escape
+import numpy as np
+
+policies = ["IN_ALL_LANGS", "IN_ANY_LANG"]
+
+"""
+This file contains a set of tools for processing the Wikipedia multilingual documents.
+In what follows, it is assumed that you have already downloaded a Wikipedia dump (https://dumps.wikimedia.org/)
+and have processed each document to clean their texts with one of the tools:
+    - https://github.com/aesuli/wikipediatools (Python 2)
+    - https://github.com/aesuli/wikipedia-extractor (Python 3)
+It is also assumed you have dowloaded the all-entities json file (e.g., https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.bz2)
+
+This tools help you in:
+    - Processes the huge json file as a stream, and create a multilingual map of corresponding titles for each language.
+    Set the policy = "IN_ALL_LANGS" will extract only titles which appear in all (AND) languages, whereas "IN_ANY_LANG"
+    extracts all titles appearing in at least one (OR) language (warning: this will creates a huge dictionary).
+    Note: This version is quite slow. Although it is run once for all, you might be prefer to take a look at "Wikidata in BigQuery".
+    - Processes the huge json file as a stream a creates a simplified file which occupies much less and is far faster to be processed.
+    - Use the multilingual map to extract, from the clean text versions, individual xml documents containing all
+    language-specific versions from the document.
+    - Fetch the multilingual documents to create, for each of the specified languages, a list containing all documents,
+    in a way that the i-th element from any list refers to the same element in the respective language.
+"""
+
+def _doc_generator(text_path, langs):
+    dotspace = re.compile(r'\.(?!\s)')
+    for l,lang in enumerate(langs):
+        print("Processing language <%s> (%d/%d)" % (lang, l, len(langs)))
+        lang_dir = join(text_path, lang)
+        split_dirs = list_dirs(lang_dir)
+        for sd,split_dir in enumerate(split_dirs):
+            print("\tprocessing split_dir <%s> (%d/%d)" % (split_dir, sd, len(split_dirs)))
+            split_files = list_files(join(lang_dir, split_dir))
+            for sf,split_file in enumerate(split_files):
+                print("\t\tprocessing split_file <%s> (%d/%d)" % (split_file, sf, len(split_files)))
+                with BZ2File(join(lang_dir, split_dir, split_file), 'r', buffering=1024*1024) as fi:
+                    while True:
+                        doc_lines = list(islice(fi, 3))
+                        if doc_lines:
+                            # some sentences are not followed by a space after the dot
+                            doc_lines[1] = dotspace.sub('. ', doc_lines[1])
+                            # [workaround] I found &nbsp; html symbol was not treated, and unescaping it now might not help...
+                            doc_lines[1] = escape(doc_lines[1].replace("&nbsp;", " "))
+                            yield doc_lines, lang
+                        else: break
+
+def _extract_title(doc_lines):
+    m = re.search('title="(.+?)"', doc_lines[0])
+    if m: return m.group(1).decode('utf-8')
+    else: raise ValueError("Error in xml format: document head is %s" % doc_lines[0])
+
+def _create_doc(target_file, id, doc, lang):
+    doc[0] = doc[0][:-2] + (' lang="%s">\n'%lang)
+    with open(target_file, 'w') as fo:
+        fo.write('<multidoc id="%s">\n'%id)
+        [fo.write(line) for line in doc]
+        fo.write('</multidoc>')
+
+def _append_doc(target_file, doc, lang):
+    doc[0] = doc[0][:-2] + (' lang="%s">\n' % lang)
+    with open(target_file, 'r', buffering=1024*1024) as fi:
+        lines = fi.readlines()
+    if doc[0] in lines[1::3]:
+        return
+    lines[-1:-1]=doc
+    with open(target_file, 'w', buffering=1024*1024) as fo:
+        [fo.write(line) for line in lines]
+
+def extract_multilingual_documents(inv_dict, langs, text_path, out_path):
+    if not os.path.exists(out_path):
+        os.makedirs(out_path)
+    for lang in langs:
+        if lang not in inv_dict:
+            raise ValueError("Lang %s is not in the dictionary" % lang)
+
+    docs_created = len(list_files(out_path))
+    print("%d multilingual documents found." % docs_created)
+    for doc,lang in _doc_generator(text_path, langs):
+        title = _extract_title(doc)
+
+        if title in inv_dict[lang]:
+            #pass
+            ids = inv_dict[lang][title]
+            for id in ids:
+                target_file = join(out_path, id) + ".xml"
+                if os.path.exists(target_file):
+                    _append_doc(target_file, doc, lang)
+                else:
+                    _create_doc(target_file, id, doc, lang)
+                    docs_created+=1
+        else:
+            if not re.match('[A-Za-z]+', title):
+                print("Title <%s> for lang <%s> not in dictionary" % (title, lang))
+
+
+
+def extract_multilingual_titles_from_simplefile(data_dir, filename, langs, policy="IN_ALL_LANGS", return_both=True):
+    simplified_file = join(data_dir,filename)
+
+    if policy not in policies:
+        raise ValueError("Policy %s not supported." % policy)
+    print("extracting multilingual titles with policy %s (%s)" % (policy,' '.join(langs)))
+
+    lang_prefix = list(langs)
+    lang_prefix.sort()
+    pickle_prefix = "extraction_" + "_".join(lang_prefix) + "." + policy
+    pickle_dict = join(data_dir, pickle_prefix+".multi_dict.pickle")
+    pickle_invdict = join(data_dir, pickle_prefix+".multi_invdict.pickle")
+    if os.path.exists(pickle_invdict):
+        if return_both and os.path.exists(pickle_dict):
+            print("Pickled files found in %s. Loading both (direct and inverse dictionaries)." % data_dir)
+            return pickle.load(open(pickle_dict, 'rb')), pickle.load(open(pickle_invdict, 'rb'))
+        elif return_both==False:
+            print("Pickled file found in %s. Loading inverse dictionary only." % pickle_invdict)
+            return pickle.load(open(pickle_invdict, 'rb'))
+
+    multiling_titles = {}
+    inv_dict = {lang:{} for lang in langs}
+
+    def process_entry(line):
+        parts = line.strip().split('\t')
+        id = parts[0]
+        if id in multiling_titles:
+            raise ValueError("id <%s> already indexed" % id)
+
+        titles = dict(((lang_title[:lang_title.find(':')],lang_title[lang_title.find(':')+1:].decode('utf-8')) for lang_title in parts[1:]))
+        for lang in titles.keys():
+            if lang not in langs:
+                del titles[lang]
+
+        if (policy == "IN_ALL_LANGS" and len(titles) == len(langs))\
+                or (policy == "IN_ANY_LANG" and len(titles) > 0):
+            multiling_titles[id] = titles
+            for lang, title in titles.items():
+                if title in inv_dict[lang]:
+                    inv_dict[lang][title].append(id)
+                inv_dict[lang][title] = [id]
+
+    with BZ2File(simplified_file, 'r', buffering=1024*1024*16) as fi:
+        completed = 0
+        try:
+            for line in fi:
+                process_entry(line)
+                completed += 1
+                if completed % 10 == 0:
+                    print("\rCompleted %d\ttitles %d" % (completed,len(multiling_titles)), end="")
+            print("\rCompleted %d\t\ttitles %d" % (completed, len(multiling_titles)), end="\n")
+        except EOFError:
+            print("\nUnexpected file ending... saving anyway")
+
+        print("Pickling dictionaries in %s" % data_dir)
+        pickle.dump(multiling_titles, open(pickle_dict,'wb'), pickle.HIGHEST_PROTOCOL)
+        pickle.dump(inv_dict, open(pickle_invdict, 'wb'), pickle.HIGHEST_PROTOCOL)
+        print("Done")
+
+    return (multiling_titles, inv_dict) if return_both else inv_dict
+
+
+# in https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.bz2
+def simplify_json_file(data_dir, langs, policy="IN_ALL_LANGS", json_file = "latest-all.json.bz2"):
+    latest_all_json_file = join(data_dir,json_file)
+
+    if policy not in policies:
+        raise ValueError("Policy %s not supported." % policy)
+
+    print("extracting multilingual titles with policy %s (%s)" % (policy,' '.join(langs)))
+
+    lang_prefix = list(langs)
+    lang_prefix.sort()
+    simple_titles_path = join(data_dir, "extraction_" + "_".join(lang_prefix) + "." + policy)
+
+    def process_entry(last, fo):
+        global written
+        id = last["id"]
+        titles = None
+        if policy == "IN_ALL_LANGS" and langs.issubset(last["labels"].keys()):
+            titles = {lang: last["labels"][lang]["value"] for lang in langs}
+        elif policy == "IN_ANY_LANG":
+            titles = {lang: last["labels"][lang]["value"] for lang in langs if lang in last["labels"]}
+
+        if titles:
+            fo.write((id+'\t'+'\t'.join([lang+':'+titles[lang] for lang in titles.keys()])+'\n').encode('utf-8'))
+            return True
+        else:
+            return False
+
+    written = 0
+    with BZ2File(latest_all_json_file, 'r', buffering=1024*1024*16) as fi, \
+            BZ2File(join(data_dir,simple_titles_path+".simple.bz2"),'w') as fo:
+        builder = ObjectBuilder()
+        completed = 0
+        for event, value in ijson.basic_parse(fi, buf_size=1024*1024*16):
+             builder.event(event, value)
+             if len(builder.value)>1:
+                if process_entry(builder.value.pop(0), fo): written += 1
+                completed += 1
+                print("\rCompleted %d\ttitles %d" % (completed,written), end="")
+        print("")
+
+        #process the last entry
+        process_entry(builder.value.pop(0))
+
+    return simple_titles_path
+
+"""
+Reads all multi-lingual documents in a folder (see wikipedia_tools.py to generate them) and generates, for each of the
+specified languages, a list contanining all its documents, so that the i-th element of any list refers to the language-
+specific version of the same document. Documents are forced to contain version in all specified languages and to contain
+a minimum number of words; otherwise it is discarded.
+"""
+class MinWordsNotReached(Exception): pass
+class WrongDocumentFormat(Exception): pass
+
+def _load_multilang_doc(path, langs, min_words=100):
+    import xml.etree.ElementTree as ET
+    from xml.etree.ElementTree import Element, ParseError
+    try:
+        root = ET.parse(path).getroot()
+        doc = {}
+        for lang in langs:
+            doc_body = root.find('.//doc[@lang="' + lang + '"]')
+            if isinstance(doc_body, Element):
+                n_words = len(doc_body.text.split(' '))
+                if n_words >= min_words:
+                    doc[lang] = doc_body.text
+                else:
+                    raise MinWordsNotReached
+            else:
+                raise WrongDocumentFormat
+    except ParseError:
+        raise WrongDocumentFormat
+    return doc
+
+#returns the multilingual documents mapped by language, and a counter with the number of documents readed
+def fetch_wikipedia_multilingual(wiki_multi_path, langs, min_words=100, deletions=False, max_documents=-1, pickle_name=None):
+    if pickle_name and os.path.exists(pickle_name):
+        print("unpickling %s" % pickle_name)
+        return pickle.load(open(pickle_name, 'rb'))
+
+    multi_docs = list_files(wiki_multi_path)
+    mling_documents = {l:[] for l in langs}
+    valid_documents = 0
+    minwords_exception = 0
+    wrongdoc_exception = 0
+    for d,multi_doc in enumerate(multi_docs):
+        print("\rProcessed %d/%d documents, valid %d/%d, few_words=%d, few_langs=%d" %
+              (d, len(multi_docs), valid_documents, len(multi_docs), minwords_exception, wrongdoc_exception),end="")
+        doc_path = join(wiki_multi_path, multi_doc)
+        try:
+            m_doc = _load_multilang_doc(doc_path, langs, min_words)
+            valid_documents += 1
+            for l in langs:
+                mling_documents[l].append(m_doc[l])
+        except MinWordsNotReached:
+            minwords_exception += 1
+            if deletions: os.remove(doc_path)
+        except WrongDocumentFormat:
+            wrongdoc_exception += 1
+            if deletions: os.remove(doc_path)
+        if max_documents>0 and valid_documents>=max_documents:
+            break
+
+    if pickle_name:
+        print("Pickling wikipedia documents object in %s" % pickle_name)
+        pickle.dump(mling_documents, open(pickle_name, 'wb'), pickle.HIGHEST_PROTOCOL)
+
+    return mling_documents
+
+def random_wiki_sample(l_wiki, max_documents):
+    if max_documents == 0: return None
+    langs = list(l_wiki.keys())
+    assert len(np.unique([len(l_wiki[l]) for l in langs])) == 1, 'documents across languages do not seem to be aligned'
+    ndocs_per_lang = len(l_wiki[langs[0]])
+    if ndocs_per_lang > max_documents:
+        sel = set(np.random.choice(list(range(ndocs_per_lang)), max_documents, replace=False))
+        for lang in langs:
+            l_wiki[lang] = [d for i, d in enumerate(l_wiki[lang]) if i in sel]
+    return l_wiki
+
+
+if __name__ == "__main__":
+
+    wikipedia_home = "../Datasets/Wikipedia"
+
+    from data.languages import JRC_LANGS_WITH_NLTK_STEMMING as langs
+    langs = frozenset(langs)
+
+    simple_titles_path = simplify_json_file(wikipedia_home, langs, policy="IN_ALL_LANGS", json_file="latest-all.json.bz2")
+    _, inv_dict = extract_multilingual_titles_from_simplefile(wikipedia_home, simple_titles_path, langs, policy='IN_ALL_LANGS')
+    extract_multilingual_documents(inv_dict, langs, join(wikipedia_home,'text'),
+                                   out_path=join(wikipedia_home, 'multilingual_docs_JRC_NLTK'))
+
+
diff --git a/src/data/supervised.py b/src/data/supervised.py
new file mode 100755
index 0000000..5f97e7f
--- /dev/null
+++ b/src/data/supervised.py
@@ -0,0 +1,75 @@
+from data.tsr_function__ import get_supervised_matrix, get_tsr_matrix, information_gain, chi_square
+# from util.common import *
+from sklearn.decomposition import PCA
+import numpy as np
+
+
+def zscores(x, axis=0): #scipy.stats.zscores does not avoid division by 0, which can indeed occur
+    std = np.clip(np.std(x, ddof=1, axis=axis), 1e-5, None)
+    mean = np.mean(x, axis=axis)
+    return (x - mean) / std
+
+
+def supervised_embeddings_tfidf(X,Y):
+    tfidf_norm = X.sum(axis=0)
+    F = (X.T).dot(Y) / tfidf_norm.T
+    return F
+
+
+def supervised_embeddings_ppmi(X,Y):
+    Xbin = X>0
+    D = X.shape[0]
+    Pxy = (Xbin.T).dot(Y)/D
+    Px = Xbin.sum(axis=0)/D
+    Py = Y.sum(axis=0)/D
+    F = np.asarray(Pxy/(Px.T*Py))
+    F = np.maximum(F, 1.0)
+    F = np.log(F)
+    return F
+
+
+def supervised_embeddings_tsr(X,Y, tsr_function=information_gain, max_documents=25000):
+    D = X.shape[0]
+    if D>max_documents:
+        print(f'sampling {max_documents}')
+        random_sample = np.random.permutation(D)[:max_documents]
+        X = X[random_sample]
+        Y = Y[random_sample]
+    cell_matrix = get_supervised_matrix(X, Y)
+    F = get_tsr_matrix(cell_matrix, tsr_score_funtion=tsr_function).T
+    return F
+
+
+def get_supervised_embeddings(X, Y, max_label_space=300, binary_structural_problems=-1, method='dotn', dozscore=True):
+    print('computing supervised embeddings...')
+
+    nC = Y.shape[1]
+    if nC==2 and binary_structural_problems > nC:
+        raise ValueError('not implemented in this branch')
+
+    if method=='ppmi':
+        F = supervised_embeddings_ppmi(X, Y)
+    elif method == 'dotn':
+        F = supervised_embeddings_tfidf(X, Y)
+    elif method == 'ig':
+        F = supervised_embeddings_tsr(X, Y, information_gain)
+    elif method == 'chi2':
+        F = supervised_embeddings_tsr(X, Y, chi_square)
+
+    if dozscore:
+        F = zscores(F, axis=0)
+
+    if nC > max_label_space:
+        print(f'supervised matrix has more dimensions ({nC}) than the allowed limit {max_label_space}. '
+              f'Applying PCA(n_components={max_label_space})')
+        pca = PCA(n_components=max_label_space)
+        F = pca.fit(F).transform(F)
+
+    return F
+
+
+
+
+
+
+
diff --git a/src/data/text_preprocessor.py b/src/data/text_preprocessor.py
new file mode 100644
index 0000000..1a6e3ae
--- /dev/null
+++ b/src/data/text_preprocessor.py
@@ -0,0 +1,33 @@
+from nltk.corpus import stopwords
+from data.languages import NLTK_LANGMAP
+from nltk import word_tokenize
+from nltk.stem import SnowballStemmer
+
+
+def preprocess_documents(documents, lang):
+    tokens = NLTKStemTokenizer(lang, verbose=True)
+    sw = stopwords.words(NLTK_LANGMAP[lang])
+    return [' '.join([w for w in tokens(doc) if w not in sw]) for doc in documents]
+
+
+class NLTKStemTokenizer(object):
+
+    def __init__(self, lang, verbose=False):
+        if lang not in NLTK_LANGMAP:
+            raise ValueError('Language %s is not supported in NLTK' % lang)
+        self.verbose=verbose
+        self.called = 0
+        self.wnl = SnowballStemmer(NLTK_LANGMAP[lang])
+        self.cache = {}
+
+    def __call__(self, doc):
+        self.called += 1
+        if self.verbose:
+            print("\r\t\t[documents processed %d]" % (self.called), end="")
+        tokens = word_tokenize(doc)
+        stems = []
+        for t in tokens:
+            if t not in self.cache:
+                self.cache[t] = self.wnl.stem(t)
+            stems.append(self.cache[t])
+        return stems
\ No newline at end of file
diff --git a/src/data/tsr_function__.py b/src/data/tsr_function__.py
new file mode 100755
index 0000000..0af8690
--- /dev/null
+++ b/src/data/tsr_function__.py
@@ -0,0 +1,270 @@
+import math
+import numpy as np
+from scipy.stats import t
+from joblib import Parallel, delayed
+from scipy.sparse import csr_matrix, csc_matrix
+
+
+def get_probs(tpr, fpr, pc):
+    # tpr = p(t|c) = p(tp)/p(c) = p(tp)/(p(tp)+p(fn))
+    # fpr = p(t|_c) = p(fp)/p(_c) = p(fp)/(p(fp)+p(tn))
+    pnc = 1.0 - pc
+    tp = tpr * pc
+    fn = pc - tp
+    fp = fpr * pnc
+    tn = pnc - fp
+    return ContTable(tp=tp, fn=fn, fp=fp, tn=tn)
+
+
+def apply_tsr(tpr, fpr, pc, tsr):
+    cell = get_probs(tpr, fpr, pc)
+    return tsr(cell)
+
+
+def positive_information_gain(cell):
+    if cell.tpr() < cell.fpr():
+        return 0.0
+    else:
+        return information_gain(cell)
+
+
+def posneg_information_gain(cell):
+    ig = information_gain(cell)
+    if cell.tpr() < cell.fpr():
+        return -ig
+    else:
+        return ig
+
+
+def __ig_factor(p_tc, p_t, p_c):
+    den = p_t * p_c
+    if den != 0.0 and p_tc != 0:
+        return p_tc * math.log(p_tc / den, 2)
+    else:
+        return 0.0
+
+
+def information_gain(cell):
+    return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + \
+           __ig_factor(cell.p_fp(), cell.p_f(), cell.p_not_c()) +\
+           __ig_factor(cell.p_fn(), cell.p_not_f(), cell.p_c()) + \
+           __ig_factor(cell.p_tn(), cell.p_not_f(), cell.p_not_c())
+
+
+def information_gain_mod(cell):
+    return (__ig_factor(cell.p_tp(), cell.p_f(), cell.p_c()) + __ig_factor(cell.p_tn(), cell.p_not_f(), cell.p_not_c()))  \
+           - (__ig_factor(cell.p_fp(), cell.p_f(), cell.p_not_c()) + __ig_factor(cell.p_fn(), cell.p_not_f(), cell.p_c()))
+
+
+def pointwise_mutual_information(cell):
+    return __ig_factor(cell.p_tp(), cell.p_f(), cell.p_c())
+
+
+def gain_ratio(cell):
+    pc = cell.p_c()
+    pnc = 1.0 - pc
+    norm = pc * math.log(pc, 2) + pnc * math.log(pnc, 2)
+    return information_gain(cell) / (-norm)
+
+
+def chi_square(cell):
+    den = cell.p_f() * cell.p_not_f() * cell.p_c() * cell.p_not_c()
+    if den==0.0: return 0.0
+    num = gss(cell)**2
+    return num / den
+
+
+def relevance_frequency(cell):
+    a = cell.tp
+    c = cell.fp
+    if c == 0: c = 1
+    return math.log(2.0 + (a * 1.0 / c), 2)
+
+
+def idf(cell):
+    if cell.p_f()>0:
+        return math.log(1.0 / cell.p_f())
+    return 0.0
+
+
+def gss(cell):
+    return cell.p_tp()*cell.p_tn() - cell.p_fp()*cell.p_fn()
+
+
+def conf_interval(xt, n):
+    if n>30:
+        z2 = 3.84145882069 # norm.ppf(0.5+0.95/2.0)**2
+    else:
+        z2 = t.ppf(0.5 + 0.95 / 2.0, df=max(n-1,1)) ** 2
+    p = (xt + 0.5 * z2) / (n + z2)
+    amplitude = 0.5 * z2 * math.sqrt((p * (1.0 - p)) / (n + z2))
+    return p, amplitude
+
+def strength(minPosRelFreq, minPos, maxNeg):
+    if minPos > maxNeg:
+        return math.log(2.0 * minPosRelFreq, 2.0)
+    else:
+        return 0.0
+
+
+#set cancel_features=True to allow some features to be weighted as 0 (as in the original article)
+#however, for some extremely imbalanced dataset caused all documents to be 0
+def conf_weight(cell, cancel_features=False):
+    c = cell.get_c()
+    not_c = cell.get_not_c()
+    tp = cell.tp
+    fp = cell.fp
+
+    pos_p, pos_amp = conf_interval(tp, c)
+    neg_p, neg_amp = conf_interval(fp, not_c)
+
+    min_pos = pos_p-pos_amp
+    max_neg = neg_p+neg_amp
+    den = (min_pos + max_neg)
+    minpos_relfreq = min_pos / (den if den != 0 else 1)
+
+    str_tplus = strength(minpos_relfreq, min_pos, max_neg);
+
+    if str_tplus == 0 and not cancel_features:
+        return 1e-20
+
+    return str_tplus;
+
+
+class ContTable:
+
+    def __init__(self, tp=0, tn=0, fp=0, fn=0):
+        self.tp=tp
+        self.tn=tn
+        self.fp=fp
+        self.fn=fn
+
+    def get_d(self): return self.tp + self.tn + self.fp + self.fn
+
+    def get_c(self): return self.tp + self.fn
+
+    def get_not_c(self): return self.tn + self.fp
+
+    def get_f(self): return self.tp + self.fp
+
+    def get_not_f(self): return self.tn + self.fn
+
+    def p_c(self): return (1.0*self.get_c())/self.get_d()
+
+    def p_not_c(self): return 1.0-self.p_c()
+
+    def p_f(self): return (1.0*self.get_f())/self.get_d()
+
+    def p_not_f(self): return 1.0-self.p_f()
+
+    def p_tp(self): return (1.0*self.tp) / self.get_d()
+
+    def p_tn(self): return (1.0*self.tn) / self.get_d()
+
+    def p_fp(self): return (1.0*self.fp) / self.get_d()
+
+    def p_fn(self): return (1.0*self.fn) / self.get_d()
+
+    def tpr(self):
+        c = 1.0*self.get_c()
+        return self.tp / c if c > 0.0 else 0.0
+
+    def fpr(self):
+        _c = 1.0*self.get_not_c()
+        return self.fp / _c if _c > 0.0 else 0.0
+
+
+def round_robin_selection(X, Y, k, tsr_function=positive_information_gain):
+    print(f'[selectiong {k} terms]')
+    nC = Y.shape[1]
+    FC = get_tsr_matrix(get_supervised_matrix(X, Y), tsr_function).T
+    best_features_idx = np.argsort(-FC, axis=0).flatten()
+    tsr_values = FC.flatten()
+    selected_indexes_set = set()
+    selected_indexes = list()
+    selected_value = list()
+    from_category = list()
+    round_robin = iter(best_features_idx)
+    values_iter = iter(tsr_values)
+    round=0
+    while len(selected_indexes) < k:
+        term_idx = next(round_robin)
+        term_val = next(values_iter)
+        if term_idx not in selected_indexes_set:
+            selected_indexes_set.add(term_idx)
+            selected_indexes.append(term_idx)
+            selected_value.append(term_val)
+            from_category.append(round)
+        round = (round + 1) % nC
+    return np.asarray(selected_indexes, dtype=int), np.asarray(selected_value, dtype=float), np.asarray(from_category)
+
+
+def feature_label_contingency_table(positive_document_indexes, feature_document_indexes, nD):
+    tp_ = len(positive_document_indexes & feature_document_indexes)
+    fp_ = len(feature_document_indexes - positive_document_indexes)
+    fn_ = len(positive_document_indexes - feature_document_indexes)
+    tn_ = nD - (tp_ + fp_ + fn_)
+    return ContTable(tp=tp_, tn=tn_, fp=fp_, fn=fn_)
+
+
+def category_tables(feature_sets, category_sets, c, nD, nF):
+    return [feature_label_contingency_table(category_sets[c], feature_sets[f], nD) for f in range(nF)]
+
+
+"""
+Computes the nC x nF supervised matrix M where Mcf is the 4-cell contingency table for feature f and class c.
+Efficiency O(nF x nC x log(S)) where S is the sparse factor
+"""
+def get_supervised_matrix(coocurrence_matrix, label_matrix, n_jobs=-1):
+    nD, nF = coocurrence_matrix.shape
+    nD2, nC = label_matrix.shape
+
+    if nD != nD2:
+        raise ValueError('Number of rows in coocurrence matrix shape %s and label matrix shape %s is not consistent' %
+                         (coocurrence_matrix.shape,label_matrix.shape))
+
+    def nonzero_set(matrix, col):
+        return set(matrix[:, col].nonzero()[0])
+
+    if isinstance(coocurrence_matrix, csr_matrix):
+        coocurrence_matrix = csc_matrix(coocurrence_matrix)
+    feature_sets = [nonzero_set(coocurrence_matrix, f) for f in range(nF)]
+    category_sets = [nonzero_set(label_matrix, c) for c in range(nC)]
+    cell_matrix = Parallel(n_jobs=n_jobs, backend="threading")(delayed(category_tables)(feature_sets, category_sets, c, nD, nF) for c in range(nC))
+    return np.array(cell_matrix)
+
+# obtains the matrix T where Tcf=tsr(f,c) is the tsr score for category c and feature f
+def get_tsr_matrix(cell_matrix, tsr_score_funtion):
+    nC,nF = cell_matrix.shape
+    tsr_matrix = [[tsr_score_funtion(cell_matrix[c,f]) for f in range(nF)] for c in range(nC)]
+    return np.array(tsr_matrix)
+
+
+""" The Fisher-score [1] is not computed on the 4-cell contingency table, but can
+take as input any real-valued feature column (e.g., tf-idf weights).
+feat is the feature vector, and c is a binary classification vector.
+This implementation covers only the binary case, while the formula is defined for multiclass
+single-label scenarios, for which the version [2] might be preferred.
+[1] R.O. Duda, P.E. Hart, and D.G. Stork. Pattern classification. Wiley-interscience, 2012.
+[2] Gu, Q., Li, Z., & Han, J. (2012). Generalized fisher score for feature selection. arXiv preprint arXiv:1202.3725.
+"""
+def fisher_score_binary(feat, c):
+    neg = np.ones_like(c) - c
+
+    npos = np.sum(c)
+    nneg = np.sum(neg)
+
+    mupos = np.mean(feat[c == 1])
+    muneg = np.mean(feat[neg == 1])
+    mu = np.mean(feat)
+
+    stdpos = np.std(feat[c == 1])
+    stdneg = np.std(feat[neg == 1])
+
+    num = npos * ((mupos - mu) ** 2) + nneg * ((muneg - mu) ** 2)
+    den = npos * (stdpos ** 2) + nneg * (stdneg ** 2)
+
+    if den>0:
+        return num / den
+    else:
+        return num
diff --git a/src/dataset_builder.py b/src/dataset_builder.py
new file mode 100644
index 0000000..c28fbcf
--- /dev/null
+++ b/src/dataset_builder.py
@@ -0,0 +1,567 @@
+from os.path import join, exists
+from nltk.corpus import stopwords
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
+from sklearn.preprocessing import MultiLabelBinarizer
+from data.reader.jrcacquis_reader import *
+from data.languages import lang_set, NLTK_LANGMAP, RCV2_LANGS_WITH_NLTK_STEMMING
+from data.reader.rcv_reader import fetch_RCV1, fetch_RCV2, fetch_topic_hierarchy
+from data.reader.wikipedia_tools import fetch_wikipedia_multilingual, random_wiki_sample
+from data.text_preprocessor import NLTKStemTokenizer, preprocess_documents
+import pickle
+import numpy as np
+from sklearn.model_selection import train_test_split
+from scipy.sparse import issparse
+import itertools
+
+
+class MultilingualDataset:
+    """
+    A multilingual dataset is a dictionary of training and test documents indexed by language code.
+    Train and test sets are represented as tuples of the type (X,Y,ids), where X is a matrix representation of the
+    documents (e.g., a document-by-term sparse csr_matrix), Y is a document-by-label binary np.array indicating the
+    labels of each document, and ids is a list of document-identifiers from the original collection.
+    """
+
+    def __init__(self):
+        self.dataset_name = ""
+        self.multiling_dataset = {}
+
+    def add(self, lang, Xtr, Ytr, Xte, Yte, tr_ids=None, te_ids=None):
+        self.multiling_dataset[lang] = ((Xtr, Ytr, tr_ids), (Xte, Yte, te_ids))
+
+    def save(self, file):
+        self.sort_indexes()
+        pickle.dump(self, open(file, 'wb'), pickle.HIGHEST_PROTOCOL)
+        return self
+
+    def __getitem__(self, item):
+        if item in self.langs():
+            return self.multiling_dataset[item]
+        return None
+
+    @classmethod
+    def load(cls, file):
+        data = pickle.load(open(file, 'rb'))
+        data.sort_indexes()
+        return data
+
+    @classmethod
+    def load_ids(cls, file):
+        data = pickle.load(open(file, 'rb'))
+        tr_ids = {lang:tr_ids for (lang,((_,_,tr_ids), (_,_,_))) in data.multiling_dataset.items()}
+        te_ids = {lang: te_ids for (lang, ((_, _, _), (_, _, te_ids))) in data.multiling_dataset.items()}
+        return tr_ids, te_ids
+
+    def sort_indexes(self):
+        for (lang, ((Xtr,_,_),(Xte,_,_))) in self.multiling_dataset.items():
+            if issparse(Xtr): Xtr.sort_indices()
+            if issparse(Xte): Xte.sort_indices()
+
+    def set_view(self, categories=None, languages=None):
+        if categories is not None:
+            if isinstance(categories, int):
+                categories = np.array([categories])
+            elif isinstance(categories, list):
+                categories = np.array(categories)
+            self.categories_view = categories
+        if languages is not None:
+            self.languages_view = languages
+
+    def training(self):
+        return self.lXtr(), self.lYtr()
+
+    def test(self):
+        return self.lXte(), self.lYte()
+
+    def lXtr(self):
+        return {lang:Xtr for (lang, ((Xtr,_,_),_)) in self.multiling_dataset.items() if lang in self.langs()}
+
+    def lXte(self):
+        return {lang:Xte for (lang, (_,(Xte,_,_))) in self.multiling_dataset.items() if lang in self.langs()}
+
+    def lYtr(self):
+        return {lang:self.cat_view(Ytr) for (lang, ((_,Ytr,_),_)) in self.multiling_dataset.items() if lang in self.langs()}
+
+    def lYte(self):
+        return {lang:self.cat_view(Yte) for (lang, (_,(_,Yte,_))) in self.multiling_dataset.items() if lang in self.langs()}
+
+    def cat_view(self, Y):
+        if hasattr(self, 'categories_view'):
+            return Y[:,self.categories_view]
+        else:
+            return Y
+
+    def langs(self):
+        if hasattr(self, 'languages_view'):
+            langs = self.languages_view
+        else:
+            langs = sorted(self.multiling_dataset.keys())
+        return langs
+
+    def num_categories(self):
+        return self.lYtr()[self.langs()[0]].shape[1]
+
+    def show_dimensions(self):
+        for (lang, ((Xtr, Ytr, IDtr), (Xte, Yte, IDte))) in self.multiling_dataset.items():
+            if lang not in self.langs(): continue
+            if hasattr(Xtr, 'shape') and hasattr(Xte, 'shape'):
+                print("Lang {}, Xtr={}, ytr={}, Xte={}, yte={}".format(lang, Xtr.shape, self.cat_view(Ytr).shape, Xte.shape, self.cat_view(Yte).shape))
+
+    def show_category_prevalences(self):
+        #pass
+        nC = self.num_categories()
+        accum_tr = np.zeros(nC, dtype=np.int)
+        accum_te = np.zeros(nC, dtype=np.int)
+        in_langs = np.zeros(nC, dtype=np.int) #count languages with at least one positive example (per category)
+        for (lang, ((Xtr, Ytr, IDtr), (Xte, Yte, IDte))) in self.multiling_dataset.items():
+            if lang not in self.langs(): continue
+            prev_train = np.sum(self.cat_view(Ytr), axis=0)
+            prev_test = np.sum(self.cat_view(Yte), axis=0)
+            accum_tr += prev_train
+            accum_te += prev_test
+            in_langs += (prev_train>0)*1
+            print(lang+'-train', prev_train)
+            print(lang+'-test', prev_test)
+        print('all-train', accum_tr)
+        print('all-test', accum_te)
+
+        return accum_tr, accum_te, in_langs
+
+    def set_labels(self, labels):
+        self.labels = labels
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Helpers
+# ----------------------------------------------------------------------------------------------------------------------
+def get_active_labels(doclist):
+    cat_list = set()
+    for d in doclist:
+        cat_list.update(d.categories)
+    return list(cat_list)
+
+def filter_by_categories(doclist, keep_categories):
+    catset = frozenset(keep_categories)
+    for d in doclist:
+        d.categories = list(set(d.categories).intersection(catset))
+
+def __years_to_str(years):
+    if isinstance(years, list):
+        if len(years) > 1:
+            return str(years[0])+'-'+str(years[-1])
+        return str(years[0])
+    return str(years)
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Matrix builders
+# ----------------------------------------------------------------------------------------------------------------------
+def build_independent_matrices(dataset_name, langs, training_docs, test_docs, label_names, wiki_docs=[], preprocess=True):
+    """
+    Builds the document-by-term weighted matrices for each language. Representations are independent of each other,
+    i.e., each language-specific matrix lies in a dedicate feature space.
+    :param dataset_name: the name of the dataset (str)
+    :param langs: list of languages (str)
+    :param training_docs: map {lang:doc-list} where each doc is a tuple (text, categories, id)
+    :param test_docs: map {lang:doc-list} where each doc is a tuple (text, categories, id)
+    :param label_names: list of names of labels (str)
+    :param wiki_docs: doc-list (optional), if specified, project all wiki docs in the feature spaces built for the languages
+    :param preprocess: whether or not to apply language-specific text preprocessing (stopword removal and stemming)
+    :return: a MultilingualDataset. If wiki_docs has been specified, a dictionary lW is also returned, which indexes
+    by language the processed wikipedia documents in their respective language-specific feature spaces
+    """
+
+    mlb = MultiLabelBinarizer()
+    mlb.fit([label_names])
+
+    lW = {}
+
+    multilingual_dataset = MultilingualDataset()
+    multilingual_dataset.dataset_name = dataset_name
+    multilingual_dataset.set_labels(mlb.classes_)
+    for lang in langs:
+        print("\nprocessing %d training, %d test, %d wiki for language <%s>" %
+              (len(training_docs[lang]), len(test_docs[lang]), len(wiki_docs[lang]) if wiki_docs else 0, lang))
+
+        tr_data, tr_labels, IDtr = zip(*training_docs[lang])
+        te_data, te_labels, IDte = zip(*test_docs[lang])
+
+        if preprocess:
+            tfidf = TfidfVectorizer(strip_accents='unicode', min_df=3, sublinear_tf=True,
+                                    tokenizer=NLTKStemTokenizer(lang, verbose=True),
+                                    stop_words=stopwords.words(NLTK_LANGMAP[lang]))
+        else:
+            tfidf = TfidfVectorizer(strip_accents='unicode', min_df=3, sublinear_tf=True)
+
+        Xtr = tfidf.fit_transform(tr_data)
+        Xte = tfidf.transform(te_data)
+        if wiki_docs:
+            lW[lang] = tfidf.transform(wiki_docs[lang])
+
+        Ytr = mlb.transform(tr_labels)
+        Yte = mlb.transform(te_labels)
+
+        multilingual_dataset.add(lang, Xtr, Ytr, Xte, Yte, IDtr, IDte)
+
+    multilingual_dataset.show_dimensions()
+    multilingual_dataset.show_category_prevalences()
+
+    if wiki_docs:
+        return multilingual_dataset, lW
+    else:
+        return multilingual_dataset
+
+
+# creates a MultilingualDataset where matrices shares a single yuxtaposed feature space
+def build_juxtaposed_matrices(dataset_name, langs, training_docs, test_docs, label_names, preprocess=True):
+    """
+    Builds the document-by-term weighted matrices for each language. Representations are not independent of each other,
+    since all of them lie on the same yuxtaposed feature space.
+    :param dataset_name: the name of the dataset (str)
+    :param langs: list of languages (str)
+    :param training_docs: map {lang:doc-list} where each doc is a tuple (text, categories, id)
+    :param test_docs: map {lang:doc-list} where each doc is a tuple (text, categories, id)
+    :param label_names: list of names of labels (str)
+    :param preprocess: whether or not to apply language-specific text preprocessing (stopword removal and stemming)
+    :return: a MultilingualDataset. If wiki_docs has been specified, a dictionary lW is also returned, which indexes
+    by language the processed wikipedia documents in their respective language-specific feature spaces
+    """
+
+    multiling_dataset = MultilingualDataset()
+    multiling_dataset.dataset_name = dataset_name
+
+    mlb = MultiLabelBinarizer()
+    mlb.fit([label_names])
+
+    multiling_dataset.set_labels(mlb.classes_)
+
+    tr_data_stack = []
+    for lang in langs:
+        print("\nprocessing %d training and %d test for language <%s>" % (len(training_docs[lang]), len(test_docs[lang]), lang))
+        tr_data, tr_labels, tr_ID = zip(*training_docs[lang])
+        te_data, te_labels, te_ID = zip(*test_docs[lang])
+        if preprocess:
+            tr_data = preprocess_documents(tr_data, lang)
+            te_data = preprocess_documents(te_data, lang)
+        tr_data_stack.extend(tr_data)
+        multiling_dataset.add(lang, tr_data, tr_labels, te_data, te_labels, tr_ID, te_ID)
+
+    tfidf = TfidfVectorizer(strip_accents='unicode', min_df=3, sublinear_tf=True)
+    tfidf.fit(tr_data_stack)
+
+    for lang in langs:
+        print("\nweighting documents for language <%s>" % (lang))
+        (tr_data, tr_labels, tr_ID), (te_data, te_labels, te_ID) = multiling_dataset[lang]
+        Xtr = tfidf.transform(tr_data)
+        Xte = tfidf.transform(te_data)
+        Ytr = mlb.transform(tr_labels)
+        Yte = mlb.transform(te_labels)
+        multiling_dataset.add(lang,Xtr,Ytr,Xte,Yte,tr_ID,te_ID)
+
+    multiling_dataset.show_dimensions()
+    return multiling_dataset
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Methods to recover the original documents from the MultilingualDataset's ids
+# ----------------------------------------------------------------------------------------------------------------------
+"""
+This method has been added a posteriori, to create document embeddings using the polylingual embeddings of the recent
+article 'Word Translation without Parallel Data'; basically, it takes one of the splits and retrieves the RCV documents
+from the doc ids and then pickles an object (tr_docs, te_docs, label_names) in the outpath
+"""
+def retrieve_rcv_documents_from_dataset(datasetpath, rcv1_data_home, rcv2_data_home, outpath):
+
+    tr_ids, te_ids = MultilingualDataset.load_ids(datasetpath)
+    assert tr_ids.keys() == te_ids.keys(), 'inconsistent keys tr vs te'
+    langs = list(tr_ids.keys())
+
+    print('fetching the datasets')
+    rcv1_documents, labels_rcv1 = fetch_RCV1(rcv1_data_home, split='train')
+    rcv2_documents, labels_rcv2 = fetch_RCV2(rcv2_data_home, [l for l in langs if l != 'en'])
+
+    filter_by_categories(rcv1_documents, labels_rcv2)
+    filter_by_categories(rcv2_documents, labels_rcv1)
+
+    label_names = get_active_labels(rcv1_documents + rcv2_documents)
+    print('Active labels in RCV1/2 {}'.format(len(label_names)))
+
+    print('rcv1: {} train, {} test, {} categories'.format(len(rcv1_documents), 0, len(label_names)))
+    print('rcv2: {} documents'.format(len(rcv2_documents)), Counter([doc.lang for doc in rcv2_documents]))
+
+    all_docs = rcv1_documents + rcv2_documents
+    mlb = MultiLabelBinarizer()
+    mlb.fit([label_names])
+
+    dataset = MultilingualDataset()
+    for lang in langs:
+        analyzer = CountVectorizer(strip_accents='unicode', min_df=3,
+                                   stop_words=stopwords.words(NLTK_LANGMAP[lang])).build_analyzer()
+
+        Xtr,Ytr,IDtr = zip(*[(d.text,d.categories,d.id) for d in all_docs if d.lang == lang and d.id in tr_ids[lang]])
+        Xte,Yte,IDte = zip(*[(d.text,d.categories,d.id) for d in all_docs if d.lang == lang and d.id in te_ids[lang]])
+        Xtr = [' '.join(analyzer(d)) for d in Xtr]
+        Xte = [' '.join(analyzer(d)) for d in Xte]
+        Ytr = mlb.transform(Ytr)
+        Yte = mlb.transform(Yte)
+        dataset.add(lang, Xtr, Ytr, Xte, Yte, IDtr, IDte)
+
+    dataset.save(outpath)
+
+"""
+Same thing but for JRC-Acquis
+"""
+def retrieve_jrc_documents_from_dataset(datasetpath, jrc_data_home, train_years, test_years, cat_policy, most_common_cat, outpath):
+
+    tr_ids, te_ids = MultilingualDataset.load_ids(datasetpath)
+    assert tr_ids.keys() == te_ids.keys(), 'inconsistent keys tr vs te'
+    langs = list(tr_ids.keys())
+
+    print('fetching the datasets')
+
+    cat_list = inspect_eurovoc(jrc_data_home, select=cat_policy)
+    training_docs, label_names = fetch_jrcacquis(langs=langs, data_path=jrc_data_home, years=train_years,
+                                                 cat_filter=cat_list, cat_threshold=1, parallel=None,
+                                                 most_frequent=most_common_cat)
+    test_docs, _ = fetch_jrcacquis(langs=langs, data_path=jrc_data_home, years=test_years, cat_filter=label_names,
+                                   parallel='force')
+
+    def filter_by_id(doclist, ids):
+        ids_set = frozenset(itertools.chain.from_iterable(ids.values()))
+        return [x for x in doclist if (x.parallel_id+'__'+x.id) in ids_set]
+
+    training_docs = filter_by_id(training_docs, tr_ids)
+    test_docs = filter_by_id(test_docs, te_ids)
+
+    print('jrc: {} train, {} test, {} categories'.format(len(training_docs), len(test_docs), len(label_names)))
+
+    mlb = MultiLabelBinarizer()
+    mlb.fit([label_names])
+
+    dataset = MultilingualDataset()
+    for lang in langs:
+        analyzer = CountVectorizer(strip_accents='unicode', min_df=3,
+                                   stop_words=stopwords.words(NLTK_LANGMAP[lang])).build_analyzer()
+
+        Xtr,Ytr,IDtr = zip(*[(d.text,d.categories,d.parallel_id+'__'+d.id) for d in training_docs if d.lang == lang])
+        Xte,Yte,IDte = zip(*[(d.text,d.categories,d.parallel_id+'__'+d.id) for d in test_docs if d.lang == lang])
+        Xtr = [' '.join(analyzer(d)) for d in Xtr]
+        Xte = [' '.join(analyzer(d)) for d in Xte]
+        Ytr = mlb.transform(Ytr)
+        Yte = mlb.transform(Yte)
+        dataset.add(lang, Xtr, Ytr, Xte, Yte, IDtr, IDte)
+
+    dataset.save(outpath)
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Dataset Generators
+# ----------------------------------------------------------------------------------------------------------------------
+def prepare_jrc_datasets(jrc_data_home, wiki_data_home, langs, train_years, test_years, cat_policy, most_common_cat=-1, max_wiki=5000, run=0):
+    """
+    Prepare all datasets for JRC-Acquis. The datasets include the "feature-independent" version, the
+    "feature-yuxtaposed" version, the monolingual version for the UpperBound, and the processed wikipedia matrices.
+    In all cases, training documents are strictly non-parallel, and test documents are strictly parallel
+    :param jrc_data_home: path to the raw JRC-Acquis documents (it will be downloaded if not found), and the path where
+    all splits will be generated
+    :param wiki_data_home: path to the wikipedia dump (see data/readers/wikipedia_tools.py)
+    :param langs: the list of languages to consider (as defined in data/languages.py)
+    :param train_years: a list of ints containing the years to be considered as training documents
+    :param test_years: a list of ints containing the  years to be considered as test documents
+    :param cat_policy: a string indicating which category selection policy to apply. Valid policies are, e.g., "all"
+    (select all categories), "broadest" (select only the broadest concepts in the taxonomy), or "leaves" (select the
+    leaves concepts in the taxonomy). See inspect_eurovoc from data/reader/jrcacquis_reader.py for more details
+    :param most_common_cat: the maximum number of most common categories to consider, or -1 to keep them all
+    :param max_wiki: the maximum number of wikipedia documents to consider (default 5000)
+    :param run: a numeric label naming the random split (useful to keep track of different runs)
+    :return: None
+    """
+
+    name = 'JRCacquis'
+    run = '_run' + str(run)
+    config_name = 'jrc_nltk_' + __years_to_str(train_years) + \
+                  'vs' + __years_to_str(test_years) + \
+                  '_' + cat_policy + \
+                  ('_top' + str(most_common_cat) if most_common_cat!=-1 else '') + \
+                  '_noparallel_processed'
+
+    indep_path = join(jrc_data_home, config_name + run + '.pickle')
+    upper_path = join(jrc_data_home, config_name + run + '_upper.pickle')
+    yuxta_path = join(jrc_data_home, config_name + run + '_yuxtaposed.pickle')
+    wiki_path  = join(jrc_data_home, config_name + run + '.wiki.pickle')
+    wiki_docs_path = join(jrc_data_home, config_name + '.wiki.raw.pickle')
+
+    cat_list = inspect_eurovoc(jrc_data_home, select=cat_policy)
+    training_docs, label_names = fetch_jrcacquis(langs=langs, data_path=jrc_data_home, years=train_years,
+                                                 cat_filter=cat_list, cat_threshold=1, parallel=None,
+                                                 most_frequent=most_common_cat)
+    test_docs, _ = fetch_jrcacquis(langs=langs, data_path=jrc_data_home, years=test_years, cat_filter=label_names,
+                                   parallel='force')
+
+    print('Generating feature-independent dataset...')
+    training_docs_no_parallel = random_sampling_avoiding_parallel(training_docs)
+
+    def _group_by_lang(doc_list, langs):
+        return {lang: [(d.text, d.categories, d.parallel_id + '__' + d.id) for d in doc_list if d.lang == lang]
+                for lang in langs}
+
+    training_docs = _group_by_lang(training_docs, langs)
+    training_docs_no_parallel = _group_by_lang(training_docs_no_parallel, langs)
+    test_docs = _group_by_lang(test_docs, langs)
+    if not exists(indep_path):
+        wiki_docs=None
+        if max_wiki>0:
+            if not exists(wiki_docs_path):
+                wiki_docs = fetch_wikipedia_multilingual(wiki_data_home, langs, min_words=50, deletions=False)
+                wiki_docs = random_wiki_sample(wiki_docs, max_wiki)
+                pickle.dump(wiki_docs, open(wiki_docs_path, 'wb'), pickle.HIGHEST_PROTOCOL)
+            else:
+                wiki_docs = pickle.load(open(wiki_docs_path, 'rb'))
+            wiki_docs = random_wiki_sample(wiki_docs, max_wiki)
+
+        if wiki_docs:
+            lang_data, wiki_docs = build_independent_matrices(name, langs, training_docs_no_parallel, test_docs, label_names, wiki_docs)
+            pickle.dump(wiki_docs, open(wiki_path, 'wb'), pickle.HIGHEST_PROTOCOL)
+        else:
+            lang_data = build_independent_matrices(name, langs, training_docs_no_parallel, test_docs, label_names)
+
+        lang_data.save(indep_path)
+
+    print('Generating upper-bound (English-only) dataset...')
+    if not exists(upper_path):
+        training_docs_eng_only = {'en':training_docs['en']}
+        test_docs_eng_only = {'en':test_docs['en']}
+        build_independent_matrices(name, ['en'], training_docs_eng_only, test_docs_eng_only, label_names).save(upper_path)
+
+    print('Generating yuxtaposed dataset...')
+    if not exists(yuxta_path):
+        build_juxtaposed_matrices(name, langs, training_docs_no_parallel, test_docs, label_names).save(yuxta_path)
+
+
+def prepare_rcv_datasets(outpath, rcv1_data_home, rcv2_data_home, wiki_data_home, langs,
+                         train_for_lang=1000, test_for_lang=1000, max_wiki=5000, preprocess=True, run=0):
+    """
+        Prepare all datasets for RCV1/RCV2. The datasets include the "feature-independent" version, the
+        "feature-yuxtaposed" version, the monolingual version for the UpperBound, and the processed wikipedia matrices.
+
+        :param outpath: path where all splits will be dumped
+        :param rcv1_data_home: path to the RCV1-v2 dataset (English only)
+        :param rcv2_data_home: path to the RCV2 dataset (all languages other than English)
+        :param wiki_data_home: path to the wikipedia dump (see data/readers/wikipedia_tools.py)
+        :param langs: the list of languages to consider (as defined in data/languages.py)
+        :param train_for_lang: maximum number of training documents per language
+        :param test_for_lang:  maximum number of test documents per language
+        :param max_wiki: the maximum number of wikipedia documents to consider (default 5000)
+        :param preprocess: whether or not to apply language-specific preprocessing (stopwords removal and stemming)
+        :param run: a numeric label naming the random split (useful to keep track of different runs)
+        :return: None
+        """
+
+    assert 'en' in langs, 'English is not in requested languages, but is needed for some datasets'
+    assert len(langs)>1, 'the multilingual dataset cannot be built with only one dataset'
+    assert not preprocess or set(langs).issubset(set(RCV2_LANGS_WITH_NLTK_STEMMING+['en'])), \
+        "languages not in RCV1-v2/RCV2 scope or not in valid for NLTK's processing"
+
+    name = 'RCV1/2'
+    run = '_run' + str(run)
+    config_name = 'rcv1-2_nltk_trByLang'+str(train_for_lang)+'_teByLang'+str(test_for_lang)+\
+                  ('_processed' if preprocess else '_raw')
+
+    indep_path = join(outpath, config_name + run + '.pickle')
+    upper_path = join(outpath, config_name + run +'_upper.pickle')
+    yuxta_path = join(outpath, config_name + run +'_yuxtaposed.pickle')
+    wiki_path = join(outpath, config_name + run + '.wiki.pickle')
+    wiki_docs_path = join(outpath, config_name + '.wiki.raw.pickle')
+
+    print('fetching the datasets')
+    rcv1_documents, labels_rcv1 = fetch_RCV1(rcv1_data_home, split='train')
+    rcv2_documents, labels_rcv2 = fetch_RCV2(rcv2_data_home, [l for l in langs if l!='en'])
+    filter_by_categories(rcv1_documents, labels_rcv2)
+    filter_by_categories(rcv2_documents, labels_rcv1)
+
+    label_names = get_active_labels(rcv1_documents+rcv2_documents)
+    print('Active labels in RCV1/2 {}'.format(len(label_names)))
+
+    print('rcv1: {} train, {} test, {} categories'.format(len(rcv1_documents), 0, len(label_names)))
+    print('rcv2: {} documents'.format(len(rcv2_documents)), Counter([doc.lang for doc in rcv2_documents]))
+
+    lang_docs = {lang: [d for d in rcv1_documents + rcv2_documents if d.lang == lang] for lang in langs}
+
+    # for the upper bound there are no parallel versions, so for the English case, we take as many documents as there
+    # would be in the multilingual case -- then we will extract from them only train_for_lang for the other cases
+    print('Generating upper-bound (English-only) dataset...')
+    train, test = train_test_split(lang_docs['en'], train_size=train_for_lang*len(langs), test_size=test_for_lang, shuffle=True)
+    train_lang_doc_map = {'en':[(d.text, d.categories, d.id) for d in train]}
+    test_lang_doc_map = {'en':[(d.text, d.categories, d.id) for d in test]}
+    build_independent_matrices(name, ['en'], train_lang_doc_map, test_lang_doc_map, label_names).save(upper_path)
+
+    train_lang_doc_map['en'] = train_lang_doc_map['en'][:train_for_lang]
+    for lang in langs:
+        if lang=='en': continue # already split
+        test_take = min(test_for_lang, len(lang_docs[lang])-train_for_lang)
+        train, test = train_test_split(lang_docs[lang], train_size=train_for_lang, test_size=test_take, shuffle=True)
+        train_lang_doc_map[lang] = [(d.text, d.categories, d.id) for d in train]
+        test_lang_doc_map[lang]  = [(d.text, d.categories, d.id) for d in test]
+
+    print('Generating feature-independent dataset...')
+    wiki_docs=None
+    if max_wiki>0:
+        if not exists(wiki_docs_path):
+            wiki_docs = fetch_wikipedia_multilingual(wiki_data_home, langs, min_words=50, deletions=False)
+            wiki_docs = random_wiki_sample(wiki_docs, max_wiki)
+            pickle.dump(wiki_docs, open(wiki_docs_path, 'wb'), pickle.HIGHEST_PROTOCOL)
+        else:
+            wiki_docs = pickle.load(open(wiki_docs_path, 'rb'))
+        wiki_docs = random_wiki_sample(wiki_docs, max_wiki)
+
+    if wiki_docs:
+        lang_data, wiki_docs_matrix = build_independent_matrices(name, langs, train_lang_doc_map, test_lang_doc_map, label_names, wiki_docs, preprocess)
+        pickle.dump(wiki_docs_matrix, open(wiki_path, 'wb'), pickle.HIGHEST_PROTOCOL)
+    else:
+        lang_data = build_independent_matrices(name, langs, train_lang_doc_map, test_lang_doc_map, label_names, wiki_docs, preprocess)
+
+    lang_data.save(indep_path)
+
+    print('Generating yuxtaposed dataset...')
+    build_juxtaposed_matrices(name, langs, train_lang_doc_map, test_lang_doc_map, label_names, preprocess).save(yuxta_path)
+
+
+#-----------------------------------------------------------------------------------------------------------------------
+# MAIN BUILDER
+#-----------------------------------------------------------------------------------------------------------------------
+
+if __name__=='__main__':
+    import sys
+
+    assert len(sys.argv) == 5, "wrong number of arguments; required: " \
+                               "<JRC_PATH> <RCV1_PATH> <RCV2_PATH> <WIKI_PATH> "
+
+    JRC_DATAPATH = sys.argv[1] # "../Datasets/JRC_Acquis_v3"
+    RCV1_PATH = sys.argv[2] #'../Datasets/RCV1-v2/unprocessed_corpus'
+    RCV2_PATH = sys.argv[3] #'../Datasets/RCV2'
+    WIKI_DATAPATH = sys.argv[4] #"../Datasets/Wikipedia/multilingual_docs_JRC_NLTK"
+
+    langs = lang_set['JRC_NLTK']
+    max_wiki = 5000
+
+    for run in range(0,10):
+        print('Building JRC-Acquis datasets run', run)
+        prepare_jrc_datasets(JRC_DATAPATH, WIKI_DATAPATH, langs,
+                             train_years=list(range(1958, 2006)), test_years=[2006], max_wiki=max_wiki,
+                             cat_policy='all', most_common_cat=300, run=run)
+
+        print('Building RCV1-v2/2 datasets run', run)
+        prepare_rcv_datasets(RCV2_PATH, RCV1_PATH, RCV2_PATH, WIKI_DATAPATH, RCV2_LANGS_WITH_NLTK_STEMMING + ['en'],
+                             train_for_lang=1000, test_for_lang=1000, max_wiki=max_wiki, run=run)
+
+        # uncomment this code if you want to retrieve the original documents to generate the data splits for PLE
+        # (make sure you have not modified the above parameters, or adapt the following paths accordingly...)
+        # datasetpath = join(RCV2_PATH,'rcv1-2_nltk_trByLang1000_teByLang1000_processed_run{}.pickle'.format(run))
+        # outpath = datasetpath.replace('_nltk_','_doclist_')
+        # retrieve_rcv_documents_from_dataset(datasetpath, RCV1_PATH, RCV2_PATH, outpath)
+
+        # datasetpath = join(JRC_DATAPATH, 'jrc_nltk_1958-2005vs2006_all_top300_noparallel_processed_run{}.pickle'.format(run))
+        # outpath = datasetpath.replace('_nltk_', '_doclist_')
+        # retrieve_jrc_documents_from_dataset(datasetpath, JRC_DATAPATH, train_years=list(range(1958, 2006)), test_years=[2006], cat_policy='all', most_common_cat=300, outpath=outpath)
+
+
+
diff --git a/src/learning/__pycache__/learners.cpython-37.pyc b/src/learning/__pycache__/learners.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..30ddb23f3d8221f695364d7faef836bf44efe930
GIT binary patch
literal 27631
zcmc(ITW}m#dR}+Wt!DrXKoGoBREyF|#4ZU^T3OOUq?Qy#u69?#Dk3d%$C5{b=>{<1
zU<T}Nkic+ey|vI<R4QD@vCEh8dKa)0$D6opmn*eN<slEbS1Qg!DtU0H%1J5}$8@Dq
zmCKVWF6aCH)3+Ib1gMoZVD*{4oIdCDIsg4~`0B_=NyFbOKlqRD{Y6XD{tGXnzbsD9
z<M93m3R|=FrdHAA*QglyHJX{FY$dyttK^pQmAuY6nPy?hte8v1N>Q$}&C=3PWk}9*
z&Echy%7~ojabB*J<-E`wT^g&5E$yl7k!!O#zO=Wp7w1L0^nq5Hu#eiq_Q(f%W#5-H
zyKIkspxLAAMrA*)#_T<~+JmbDxEi<j;%YCh4&rLU-iNDwxH^QZ{q_M|9l+ILTphFz
z;pz~sj^OIBeFRrWoCE8b%28XtqD?+_AK$FaOlE@ewYF<tTAFoiyV08Wf-xsLuP)Vm
zx3L<Ou3fwoo=+Nq+3>vOn(H}1sqVSaeLdLwwU+aa_qOBt7n?QDYs@tqcTx|AzvMRT
zD^AU=FI;>F*SW7=ZZ~bV^H$BRHJeT|DA-Q3w&vJ)I(l`kVb8te)cv;G=s0d&=P>cd
zUjxTQ9Nu9RzSh;&b$5SPThu+28^$Io*&4o0$7i`Y)CtBHYF^cAyS`&r8!fw0cf4Sj
z*U=Ta)UW?QbH`Bo&(ED+XfHXZYc1P#YNx$hjpoYoa{ILJc#Z1ZN~`5GF+rz2w|=^b
z?`iQuP8`>pT3!oArKf6RuBzr|m_sFuX?j_A$MJjrFY#B;@D~2~hzGAd8%5W!^&5t3
zeW-nC%o(;}XFf3RY8%?7_8LC*istL;YF+PTd;_(!_w=xC&dz^eb~CKEu79L|sL$zk
zf$#g7Zf0Fq@5|(vGxs@{@x*dcIahme$iy}<vG0xd^t<S-s~<v(UHun~o7pRBmSn21
ze8$w7Alt09=6T+{9t`<zt>w+N-6hABI_@5{_~)gwn03G2URr)ddSG{Xt#i7%*q&`R
zW~=dA)uolD-{1^Soo%-3wWjyVRPrv?Dc|BdUf;kueV^yRkD<V@_xE&PyNfA^hW-v<
z;wEreL51!Pq28q7?!%9}U!5P|`9T&00yXO5*?FhstS-B+u#(3?XSL4M?gkyb8jq9<
z%#jXE@w-M(?-^>OU1Q4>XJnLt-cSPtZp3p{^Be7!myFjvgx2{TGSa*@nGLcxoHY*%
z)bM<l6fh_{traXy&37iV>T~78iAq6^>!{iT=E(iWa9>s9G6qK;^69ML36<4N{h%?d
zAJg6AxL3y!|8WY><M5tG;Ru}tjn!=f^e<y)>@0q>Ho;h{<N&%Re)A!HEZ9R>zhMvu
zw{fdcYrgF)3W10*i4)8YF0uozV%Jy%IX?uDsOnuVzA@uY;67kA=PlHhogiDU`5vfW
z)^nP3?kJuG`kI>27z~2UT*E)YHo1D5KCE~44h&%mHR|k|fBa=l83qBz*EWDWi}+3s
zWic~nY?Ah3FhoJDlF?cYav*bU&wUDa+|Tf~;XciCj%jc}du*rP@UZx#M|DC00snvF
z&^k{H4sA=_k3Ud%l7smyi(QSgu<Sa-`^WjlNsjKo;ON2{KfW=IggsW5-S%wFibwX?
z14gFm#YoH*;~|U7Rc<JvB1<%>t2GnQN#JK_7TmB3W?5VEo`^d<GT33N*36_CnAK{l
zw&YZ+L8)3@YTGMKo)1^6H&<#+^~62F);`DL6pJYqr&%N;h=<`mi3h}UK~hsUGdS>1
z?$>cY{v#hakHdQlMS`S2&<Q(l7a*WyZPP9aite*Z_7LvmMRd_B1qdJ%UG$1+9}uxc
zuN0NQQYqO6!2V$H_+{auB=T>yn``{VCne%St%?~gS@^rU(D1B!EJs*aGvd5wIkozN
zWw+~~Wi8*btwzglTQ!RWY^65uoC243mK_v$I_K2<71y!Aj_Zzfd%?lEdcT3UiAL63
z!$L>?f+M{~7-M<$HtH@fG^))L_Q`r<(rV9HwU$*|Sp|!#xodr6V>JuJxzTcL3yp%b
zIP+~*YFJ>1@o+tiHXf<fd|CQL-!1jBhj%epOiI&9x6dXZDD^kvbAsf<PFfAub-ZOv
zPve%ufuhNVYe9gTZ?rhpXmX@KMF@ioIIK&CR5wxZkG})}0r0<ulWq>;NY)$kAqlVR
z-_pBzJ98tC+0yK6c%8eE@m>Ih)4dmPH!pWVaH|kbP(t)TneAp#=KMS;Anq<^y9K*&
zk53BQ+%vm5JSlbyC`(<uQysER08g=YKNG?R;bU5m3d`pL!F))X)ld?f=GR-{g|*^g
zMKI~SCv{J;E*{@<nlc-yww4^fwsR|<Q*XEIhcuHKl8n*uAs$X!$+>_;dXVAx0MsNo
zinVP!xTKBUwAVQ$;4#?J9HehZ!(mU9Bc`p!ob@c{NKF(-juXPOphgV!DWLSM+YNs~
zz)}DoKx^4GzXl|+2YUphvB_!c8Js<10blXkn58Hp3o~2u((vw`w4P~E0F<g&L8%m>
zX3Y4ote)@G?2`cW!5)>QIBh}VnZ<GgB{+4|HP9Ahd)tfOc*bkYFSQ$XUuWUH5SOei
zeaCOdWQpr3-#_aBT`^=*15S#9yjHE5n!J}e;qw>=Km!#(SAh5@rE2`1=lG;d-g2{n
z6HuOH$V;`>8i1&2d$@~dV6<on6M?yxM#516^cCeJDTM&uCNj)|Dz#gDp#>=t0~CVD
z)<9#0XOk>)Nl41#4;7dIG$cfhALcqs;b+QlrWD!s7TPOluF<OF6T{`BOpFz1_-oOm
zs-XcGR$86|$m8ry#r=tZ)OiA9ZU|LNR#JU9WOL9vsEiqx0%av95EB%_mFtYIv~IN8
zw@Hfy_@q|n3*pr{{9^DoR~jIkGG!6Lz<3sFprV+<+1e~<y}x$us=MNV!IOi|Kng8i
z@I1$*>~+a?+iozFs$1DJ2Z6hfAO*u|!cv(?DkSI}j7pu<jl(lF5c6%BWH8c4DT1-S
z%j)b(FwQyZ$8i~bPzb+&(iCw86}Qk&_a%I0urKu>8I}75Jaf;nc!dRLBqeZ&q5$Cq
zfbUnU6c#<IRodv7KBn(C^1Avrp-;$J=fn=uz*N*!Ea&IZS}b5!Z~=A#IUhRnL|2EX
zE#fuKz<xHNBg<0*Pv2GwMlZW+bu$|o>eKH{^m1TKnQl%rT>PHZZcbm*x|u7Qodd(k
z*IJP<LMHJV1aCV8mJq*0eh9^IQngvfsx1Rw9sA`7mqJ++=uUJQBy?5{lmbhlXd&<|
zA!J)f4oOBEjkvcPo)T74tg8<K6bHmDB_f4`g$!|`W<40HCS%kKS!?t9v~`D)!?dyj
z_4tW`|4B{Pfqd0c9VU|kLZqfa@rZu0o8r)*4ih&hSnB=+7Pe39rc`nThes6-IwN#^
zEVeeJZR!WKBYo<fC@Mh)<bjr*)dA>~rmvzArIVc95df2qN{kXAiJ5^KEm$D3w}|4@
z#U}Gg6L^`uP?N6gX;$Me7P-=sVTe{gFuq_Y#h1T?mqnS?%#vO*a6o1+19>{1dxUY<
zIeGjW!NGG#QJVMPQP?^#AY&UhO!xR*h<N<%0*4H(m-z^W&R&j!9u-)c);3{O(M5^f
z$g<X^p+mj3bG+;4R2|r*Ah2yv(1Yl=X?%o5TFfjKdJyL1E-X}=M$cS*shbA|p6OA{
zD<0B%rLM_eSiFpH8{N{JaR^47{yV_VZ(h^<;zmhqThvBOwz94R(}(1~2Ah}JcYnA$
z)E%Cq?oq7$Jhbx|(B5*PR>2r3Xn9MF2_dhftk8yF;WhxNSpJX;G*I^h*l;~!Xenuu
z>@`MWXbKcK;+~?VS1JV?N#XHi#4$xdK|7+NP{p&YjxflCBgSayyE`MM&}^&u^`vgI
zHtRIK?T|REd!4YpP0Jw)$iQbJIi<y^Q3@FeTl2c_ZY3aJPq&!r-}7tr8<c7Is)vuo
zl-ZDkI5nvrIC@YJC_W&p@($R2Jg?K+(;i9}P5G2Qb%~@4omG@b3<@+Sx*CKb-iAN~
ztFzOBlBf_=T8T7DYF032*W<3Y%0m49MEq*XhePY)j7KA9Cmv)2=%vlIxT&i%mpeyc
zLJ~=gLkXv~4U6*Bly#@qIT+rFWXXuGZ?%Jpb&(Wu9%%aBFbue2zWw<vrrVVDQ-|rc
zqKHsvt);<GthKA86SQc-TKmNv3C_IZS3`(RthaGnF*T1YWFK#=c{Q`ic-_kw-&!)z
z$)O;(?7}1v<Y|b8_ktwZ6<RD76R}6VhI{mKEc<KKP*n$dyJuOIcCH|QOTcY%L`g|m
zE-y@}fX7N+0hdeSTp4$r<yyn7s-{FRb;Zo{6+X>1D-gzn%vbWNW}sIBqod4>F!Qky
z_Yw;corbE(EPfq#z2hjfp)91QlAblj^|E2Y23dyrrVMsq>L-kY`oZ0>gQVS9l%jR0
zfJ4~9Sx7n*mNo!0cXc7{uvu<UfXXVu4-$`ax+Lxy9~tw2F|vtX7VZZf&I;IX?VLC(
z^2%A!%dP%%A7p<KmI9Faf6&YOhMh-?AlwJoS9TLN1sz7T23soh%<!z(E3Li{H;1(I
z-qqd^>;Xftp5_l}u$K?_MgY~k9>!7V4xJ^Vh^|e!R?kQ9yqJ2<>k_YT{vw1tZNtRd
zU^?B>W)|(1d!rBxi{0`jnOqiUqddF$E?7=!V+d@EE*G9*K6<ccETWC-FvQ2KuorWt
zvq!0Fpg(tdN;|tsCszy3AZQUDRFj33xw)p(hYPg12rkg#D~^|A;=wSu5_G=c>ris8
z`kXKT9)dEA*=)Sep`q;e!0WXj<K37`Aq$-Z>o_8;zVrxKT~!KRg4?lIr}M=2^vgPN
zr+4~JZxVD&nchbfh6x_Nj$yc8W+B={k!M8JAO|PXyyMdT7~~;F!jh+K&tGK~BJ$*z
z60pQeA+p>IAAJo4P>#b4vMej6TmW^$Goo*F&x!9jf}lZ8B-<bhU!PyeRiuG@=j;5H
z8I0T_(&10CLqXX+DU?-H)WtK87H>HHj4?sf!&HBO!X-#=RE7ADN)WCo)d8tH&pz@}
z#Bv?yj(U+G^1sFjgkyLNA=|+*D!S1zI!Bev)+*sVqs{^L`Cb8%pLV0@{_I^izWM8C
zfqc15$W=Yl&u<jE`8oI)Az6txvTIV}`W0$dd3jpEQ~DaS^fjiQ7JSpr0gdvDrI3IO
zVLZb)MsSqvychsSaXz{+1~Xql$&(y;H+PW8g#&V1L76N2kcJxIqcM;`Z$g0=N}E{4
z$veJ}BSME4cUl9$d7=fJ{lu8tyBGtMl}s~a3<6og*)~$6dpwP0ksJeJBi^iYz_Uy+
zO#L#lU<Rd>-cCh*au1*zc5j7oL@@WKSO`lLHc8&+3Z5nC`4sL2``&D~+FOkEVkuB2
z0k$9HL@pA~#AI<0M`iBclWLH48orw#ZN8K8J{oG3SX0dd$;GeW@aRv|!1SKZjDZOr
z(~kkW_5-8JM(4?0tX$0a2t7F*7jbycp#VnUSOkHEZjuone9=7y(r)|`uHk2=4ns?U
zwwSYxd$12oW;*{N7U5#KeA=p6q>3;L(C99@GhGr@5d^2L>lVT#PKC<!me(sgy<V}x
zSH*f53Z=CADRUlG87L5tGhot!kSQ+Wq?brq?i(U?o!I7=*cWF)V;5A}*Dyaye4;Gk
z&ax1PlR#89waQ%JN(Lg2VmJw!vHA~icqR%hU&a!G;61a`l7$27Lwu4rS+=7%!E!)e
z0imOk34RH}0djX1B^Jn-39>Y)fozNLC$2G$K6I`IdBKj*g`NzS_5L0Xj}%oCioLV8
zopRqz=4KSZQkD!-DU4-C4n(C4xo2?4eU=45FDO*2wB1zQ3%n(>onR(GU(NOdJ$s)=
z6{4ue=n^xR)iLIy=6LBz^Jr;LX~^aCI*#~{Q+OVS_bQ4$58s%GaP;s|x26+bdHV9=
z=c9j*Axav&dlH}o-yZeqJ?=Cb>YNow`!WpD7AOqVC|IunU9mIHx3(oA3g%{%ABu>{
zKmHmxl9i@M2`+t<U{WXzB4afZMRjD|pGI}J%7SYwc1c)gC{v{#5o8B}lhgi>a46sm
z4V0ZHA8aPuYLlc89rNFRg^Q5?X8h1{vaaLIpbcdmmP+r0pRr*}`Ihd(j-u|uE_tw<
zg_t}Bn+t3y$|ec%6ZS~qu8Q}f24Z`()zA8Q2#48q18*D^E}rX>Q_FjpCRjAdO>Z{w
zHaWM5=DvwHBo64VUM+?;6o~BX7vVB`wWP+~%^ZZ#S;ax_+xfRK2@wHF8BF43g^zV9
zVGSt{M?y?KA@aDk6M0;-TFX;4*R8Gf;vvU20>*g6cRQbfXChX{W9KNy>+&+RY;n?p
z+4fN$F0hCkq&%WuPl%ijws9BK$JXu{O3$9uK|BeGpgcAFB=FZKq^H9LC`bys|93cv
zgpubS0)jiqA;fVakqnqnuOhktDLaQ7x%|K=;3%&w9Rn)FAECyh4>Q&m`sQ_;VbX2^
z?Fw2CS#l~W3KJ;UP38LGlSxG<NJa^=`#$evK$Vn1;00VN!H0A+$aEUZDx$*TgQS;2
ziouEGU+42DSWp~MHOW0No_XF7k|PUB;{8u?C>Kb<)GbIYC4DS6ZcHFZ>LBF*vferN
z@MVsJW4NRgTKMbHGILh~oNg4oaUY_&kP}3b(Y3LzZ)AOg&(+l?ee=4FAV83uTvVrL
zfQT52g)Je3AS!(!geF1=%`ST2eUOA==Q}Yb{iwM8AfY6R`+#wVK9<lX$1n(SR?6de
z5Hp=ap{zhFY?Cfd<$?v8bk0O>z$gTQ64K2TD0!#{3uJ7aoeTXqq3E*(NXbY&t}aY#
zN<~M|g8(U}MycxSdVjECMuH6$jL<#M2eXEXvJ-eo#E&DW_9yI#{0{+#UwjC7L_?1q
z;wjza6OBOZCkdekBN;Q1?Y~81hOO}Y#~gD^AfP-51`3icpk-wlFcEgVGD{Q|jitl~
z!qP3Af!fMcicvuDQ?N3OWn5%kc`m5Q#rR_!DrPY^u0I9TwDiu&M;ZC{sM$fYpBQR_
zbKf)i;(K>S&W#u^J}u;gJv!wRLG*&Jure}C1O1w9vQgOFKYesHlw-TJR9mgKX4hc8
z`J_>o?!8^2?*AlEg{b@TLx3rySIRm3<3L+iFrXi2|5r3v;OyCtG1wjO)xClCe`HkU
z<lpB&iSAm6s>k)tiQNu(do=wX8vA%?innrh79j=K(}=0Ks^X9o&K2hck1a_3LL8<~
zl?vRnkH?l6x}WB;Hu>yOXXVF$%2B{iBCkJ`oj$z-I}HI_?7e+>n8$>RIC>l>B90zr
zf`O9rB3t+7_VOML2)jT)@}d8WBN9zt*cn7psFy&q3iRn4J*CtiQ2^NGM^FIx%jhE@
z0Q|%weZ;56_P3)S42BNpdV7woD1fl&gkxLyxJP2voVfqP87ZSs6rdzY24L97cb=Vi
z9Wh#wo-eYk2^1>#AhIHb(rsoYeu1Jk7yb%g;8&!zDCqOFRQIVTx&&b3<0QG{UA&R#
z2LmcpCYig=^g})(Wm(!5R+UB+<Ql%S<Uw>58Ar%`LhETef;B*`m8N%|-d%r8QXs}o
zmIiEA?i|1D0t+!vQJPZm=Ul1>2cb{n=|99F3u$JudMR%nHiz)9EH<P%p2dHh{PQ@x
zS5U-es!<8!(h-iX@<g&>o=7gt6Uo~{_ApFT1$)FUi>Zo~bW9ZShZ&hjlwdu1fRPFJ
zvyrJO7Zk!s!2xp<jPFUPe|7i9CzC59%q=uN<)IXAYb)Doc_MVu`y(Lz5*k7vHozKj
zG>nKls}w^-=NRj;&xhaQ=MalS^)UlG5x*L4TI80Q8zzjY8Nb+-NFUTH`MJd*+QTV_
zvdv9>V;E4+!0i#Xu&&!V^z_?6qOy38eo@&Dc};{u&9YA)&lrw9wt0^xyljhCI53iN
zyC@;!sPFzM#tZM%VtJjBMSF2ZH*1+oir+mL1s3%l#)|iJo2|kErqc|ZjXYX-Z-Uo8
z5+wgRuz}}``|!p-yun_PDv3JsrL=S5i78&DJt`rmbMBf->!YzAnS<6D%V|1HG(b#p
z4Z&}ceFP0s#c0RDDU5T4LaE%GdXTNHHoW_k17l)>#S#5I=7C~EI{c9`&|$&SiGHCa
z$%x-(Q`G+D?SvFX5gxFKA`VVjN+$Kn5CE|OUqRc2U0Kv1mG2g0kP-$W;h?1fdh{5f
zki9?_d}({=dw|iYFGWujHS{0^P<jq~6SeCy+D|&tJEtD0qfjyLR9kJ|y4`kfFdzm?
zRFkO6Q<2F4VMMeOK%H0FnhcvnTn$XpjJ2GvEsJ@Nkv3s?POXmQD=)SJdY#8?XLe<N
z+G;Oj78@OQlX%jmuiocE+$Y9P=0hJ_bsib6@LVIP$PbMAEv1qSAq^5sAo_ApxaGLB
zh`;WbsKa2N>CPS!jC9l1<bIW0^*S1t+#E$<XL(8A930}xJN^~kd7p)tugsb|FUc*i
zaIGu_g}G+UN1PEnicU+BDcY#r77Id8uy>~Ie|c$Hfwp5y7_)2gNeRhi2&=Cl6v}Pm
zMnuR)`R+3;>MRCW4Eew)LFjp$Fxl0FhkG0$D|Gc7%amdF%0tDT;B|&)<N7et1mU~G
z6aID%JrEXR0iNKQ!Eq5sXo&$0gU?z_V9KHnFLxNFC=@SDwtI+y2;V2uE4BtsrW;yn
zgp<lw4}2@1KgVy)<3z)k(3!RU=(kvCw*M9~M$o3M-@<#wy5T(y>ZQzg@Z87W2k(J=
zU-n@VyD;X?7sAN0D55>>^%J}xP9Yy)mew<g%Y!FTY`>GaWPXAINFpH16{wSzA(lUI
zqNuvP$cZ9gaiS(TRZng|Rec)q_S3|Uww<PAN-#YiXURj^{DEJ>=jR5N{IQ3AM@+f#
zR+2PGh~XDNCV?+dR|xiCq3u<6NvMa=_QdwxrDTp+UMmjDYZuVMqls!Yyn)>gG;bIu
zrco-GPntP1Zz4DcH%I~EKVsy093E*>ET~aL8-HYfqHJ{#<_hi&9#17dB)bz_j8g(@
z{{kZv6?rgMP_bh#4gj^{4Glb|zkrc*j{)np)0khN9T@o@F8uU#SS}}7iRriI-1d@0
ziq??#QE#gF`}5(&9XOL(PHpA6Gta&V#$=xi^>)AEH=T2LxVr-5=35@fGR@gcoNBKq
z(l04V%zWuZZ`D@t)IE3SqH6S{b)o4%oRHj}H)<r>C#^57AS$+o@P4`bcCFO`w?ApU
zi9HscxqyVnW&<ga$OU@MX|>lJM5ey6Vz+NK>T>fdcb;*2^X-+B*6WUYix+X<i)}>n
z*P7?<)ZFWhTci>aObHT!%jsudeD=)Lndi^E^kV$N8n!_=cZVOYoNlUIahucDg6}VT
z)2C0*<7-!Drw~$ldJb8bv+ec`1}Jblh|_QW<dsV@S)p)F*2+Kr67Yr8lewgYbMgHe
zl2e$WN(SMaf+lhmfmcT7F9xk9T#^I;kc{gugb52clbi+!12z(YxaEW-NBODLt?dC~
zu#fF>4+rZYxsQ3k+ouDD4Gg{$<*|`|CmB$HxB3cFKH+^G%x0T`P_lQMFBaw>45!Bs
z8cyjLsR}{}r3b+mI=GHmR}#YqJfqk!$n`Ja-d~BRgjR^sd!WFa*a3ADRYu4g+Tn`x
zkQIL&C#$c*IW8Mty!S#+vPl>98wIz7C{jduAPk40I&iBac>tj~Pyi5@31xsmI=G7{
z&(bw8R6s;#hGCe)a2+6k31EkB4GKsqtbzo9`yWGJ?zdTx)(M%6ra*iNbn_uNkOU*h
z!o)ooF%pbIpsT730`6DhF@ss?CgmVXHzvx?>78a|pp`gUiKC*hDHHRjIAIvXqJ}hF
zI{v`=kd}*UnF!=uEE?f-Fe@VGBVe~{9DpVUt%q8lwpd{124eS2N)o*S^Cl1oo8K7P
z)VMhTax=RAFrOi|U>P1`gyW)>%*KeF?H29a0c`?dw@08iQq#K!ZEE#(zuYZtjMDLl
zcV0rk?oc;>Bkz6>zghW3m~Bo&_-$SSb#n;1&GL*{9uwM&5bCg8-@B&WTtu*Kb{+h(
z%Y<Kq)Q%0B8#_mv?HYHfp$1_0QD_MLo$-+(L)V+YG3(Cs)aU1VpSf>i&O|W_B*+f&
zq%a7`f>Y)ThHfF#bathQh^=Z+6f(`oahEXY5;_t0jV@imhi~vEC2{w&e3G#nORmMs
zFSA%+aRtTXu)?8_ctZeaOaLeaCV~D=#k{UKkQW%Yor=+V4WTuN1p+L2;&Ma#?Ruc!
z9>BXGC)8}7a7O5*Xhr$DM2Q*__BaB?kKq{8Pa1in^O>C>Pg;xNt%wGxEyVD)HJYWD
z4a2{4lI<fx7vKvk*DI{PE^#d9d(f*Fb6xWg^L}*x!qR|%YlLUz5UIlOYq{GsHvx56
zamKG0svaU&o>!SZ-JC!%VKgrhr<z#Zl;%F<4<kf<q?=nr(6&H$Y-vE><BZe;Tc90b
z@5TgD&SC9<ErT`>+5plmfr$X#vZ+{?UHi^{icz2<IFMJikx|6F*~k^<y_bqb5$bO>
zcMN1{*%HOVMkTh>I?*ECIcYIz*PFCvqnv)p=Set5<V^36bwSp9l3a4_Qkee|wJmX@
zNr;MK35H#p+^5t_vD*rX{HuTo5I}@CD&+Y&T)Wp;{4xtcim&3VQWAJV1}x&h1c?MU
zo?@jM3(-Ooo-)gOCs1q+8Ljit6DTU#YnPCKkzoLLrrNx&VlM{tGPdyz;>2;BXn7_|
zFyb0`ffR}T`glg-uO()bQKOwFcft!yT&##>(PR!s28VYU1zi`gh;18w(WB2+$No*+
z)FyQHz^<0%JHUwye)D!t;z;wIcasGS0R(B6eL<RA;z)N7a}U!DV-<x&j+%)Wm$=qq
z=f*}#cl8ZTMK&-pcF@+%W#6S&=KEY0)lKK+ou)n11h+MJE7#0qA<;=w(aT)-2gIfd
z5apVHB1ecR#x2ZUBptC$QN{aJ4%5VmmW6!+>T2hiT@E&e6T%1K<RT95B^01OTi^tO
zjfCmIJt&DJfC&u{LHY##3|lKBjC240Aj4(2b!cuwqWCPXT(OTN)IjE%3cex+ln%Fd
z>L}hNQ_SkW!6_!N$rKx%Q@fmC<tdLr7Of=u0i`2OGb4a8anP*8II+`sfX|Rc9v~Kc
z^mjQHLM7De&+Rmh?G)|7VBVBtgw#?IMv^53l`>uIIK|IpSTZ}`!h_2k5pS6#ct^^n
zX&x|-m_@Vb@@XAM{6~Ux9*0NcQJ>mf-nH6oV75ksR1*HBXy3ooY$W=2=l88E4eZfI
zOE(oL=%fh7k;Fw>i(q1Ha#sp$XhHikt)?q3x16Bq9J(m#(8>f$Iay|-K4o2VRJtqn
z3ZyEFa1q&gAym(WE+%rdY8h$}H0M+8xl=H!F4KmqcKtzG?d^pIHU<iNnL}3I<W#6t
z5<vLJ-&nMH8yCu50td!~+Aa*ixkn*XOEjJALSta_yez~LBq<~B4aiF82?Q5pu`zcH
zsLH)+@(^g@5)o}9H*>XfZ5Da@Vl7DJ2&e4R@h%;(H3+j5fi_iv`)AnPTBUtEPArDu
z0B{}iO<>!3d_j_Z0YrU~H$<o%msnk_0a6db2@((rGH<<cNn*l-QWAeJTpw;eL6k3|
zk;;hT;gP3{giHy*-{skPR#BTorc~NwC@J_B(7u5Be?ZWteO1fs*Yl7|VFkuvfX=3K
zpB@f@Th&IQFKI3h3A7A3FBUQFPywK)9J_ppG)NfUQ#cWZcT~kAsGz=$a96&;h=h#$
z>!_bdKoJ9^v$v0_B#6zhgntr|8n{<3ClkNJ*T%sA0A5I5jZS6Rgq^X|zEN=cgJRsy
zKaPe0iuc^34<ptQ$l*AFGV6D7!cZCpaZr-?CDOf@3BzNel>o=1M-UD#ri)TGQY#eQ
z*QE{Qo~iUxxk9iir2Z_<^BX8PkgK){siX@@Y7?0bswat8H>wDD#VThzfB0$2Q;}Nm
zY0L2OH^dx|FiV;*Fl*sNVD=z#UnGhKYBr{h;i!XR8Zma4Q2K`vcrb`_*Wdv$c{}+0
zR7QRkACuzeLS2Yo|0WqZ=%ApeaNze@U(^?2!@_~T$9vyL(P!q7mdRn0Btei^!K$QF
z>5nb_GxjIcEKmCmld#L6WGp);A7$oZ!I3BxD;RI#VtYcCQZ4!jT#rfxx4mJWG8K&s
z5Jum~aT^?1#uRFNs;b7Pvfhc1-cL{eC|U2$YrCHBhiDeN9(~~xz_TD3Ju>VBJOyME
z@RUHBK^U!I8-hj)DbCj+0;RxoTV3Ygvkwz+q-6n}#<0;j^QiL_&k33mQhyP@l7$&b
z+xV5dJ6r!QA<UrUEnh+$1Nz9hNW606!J_zoj%Eam>G@rcGcA*6XOMSC;s`Us8T=$E
zb}6ZxGb*-*9g^yP4fTGHLsIUqtWg@pmYHMbu*)Z;67e5#mVf>3uW?DcVvJG|j<CIw
zNXNuFZkWl$x(P|8<JOrfm9C(6`L38Bg2-2jn9F@Z@q+9^eesRUg9(N8v_MOHo$%q;
zapEJrAN%!CXhlYt+TdWb0P7kZB;l5fwA0}##{iUSUK}U|DX{?q?i8a)rxNxSSXo94
z!v!=P6mC1>ISKM`KFl@d-CyP15pf>S6#HDmUJZ(gwE1m59_15kjuG!HzE4`bg$=CI
zw<FQ>H}H=8n=B}Vs_f$PJo^HRb1YtAA=;)M6suK{nZqC#l`YJ{{t1`vC{DDJ2?;a$
zN0~DZYOVYfo$+{nW3lpbEa6bP;nE7o|4W=;vvKS%MjJOXf@ychtQzjStFLg+ZMZ08
zlS`0Y)PZgOC0gD?+dNhSPKsPN2XYKzES6CG!sdO98&O=*R_CORpYCOSSOI7?f^&m5
zfdZ_g%B{gn`tV)!yQL++q@K#C7_-Gpd-Omh20JmauUXX>@>4qppHvReQ;}duWsN4r
z&e0aO$(cJv=O`7qdF()!>{kjmiu<#e3im@4*c#<=o(uIN=qZ_4%mhEjOYx@sEuLLt
zae{@!SA2(Oe-lNtQ5K0j0%_tr<Wzdq-(X!K{R#ai5>xqCc0O=jQ-`uF`z!26#0fnv
zj3}W6M<Pr1ChHu1uM@8`MJleftn;fl!H!l~XXJc`Hkx?XtE{YEwCfet7VDt)v|=0v
zol(dam#siDGo8muX{+m&EqxB*Yi08!PCQ+EvAr0kEtyQQnI;qbGDfDR@HL!8F*$GW
z?wc$wu^5=Aw{UMC=SipqGv(nMK(740WIj@RSJXKR{NuXCO7c6vB+?Z4iy&x*xvGV3
z&i(G{H@gME{VMGI8Fy8n3^so3=aqGBTxDs=hvz64S^r_(FH)Hjk3a#NuE0KuYpBtH
z&7<*s)`{-Lt;A39R`e9moWBBhpZmwFXJsVMW48+W9E$9rYpxscT@7J8P|bk>^gBd-
zCw=@DTFD|#@a@&6v|*zSViNoi>lqpS5WZlTqlH*i!uByTCbX5oTW96#Fj@-T+kV%d
zrSFF$|1&l6NxjpErTkgA^Ha{OFwcJ*6cj^>vhyR14VxjyAq=pfShrpHgBA8l&{^C*
zkd@>*g|WZI%fHQnhWjADAp0HB;pDsICW4?Yu>;%IAYIPsI6(oyKTNNUyj!tw91LRz
zPqGRj>_I6xcYnYZjH>GjB@kDb{$`+G3G^-71pG3p9wWTc+Ms1=A0S>w$J3zW6Toib
zIPM{b66c`k#CL9wl-G|N`}NLf0^P!QVoWEWB#iUxyf*}{=}Q7N7cZ({Gq9c-?CMd%
z44``)6qzs!n$2yD;Fmg1DjHFOca4v<k90o|{19p_kfSY{HvF1s!K6hOv^+~MnY!}6
zDQ9@@{&01Qk+vHp@)Y!S41Eo43^T7AZ|7MGhG36MV90mSZ|)=Q8#&Y&iG>ToJI1+A
z=q$(+9ogOmh#l!CCce<zD0d6-R!;qXvSPoVj2^i`{}@^*`G}*1EqmyQW)H6$zoywE
zzmAx+@ITxrgR8tF0Te1=GA3x;{O5Nu<?ipH00W87W9br0o@llcxOkM^%hWC+33IrN
z_p1aVwhA@&i3A+__0&&n^A^H1jLcudkD|b$Byt?@r^d1K<}CwGxX!WC)wYjKY9mXY
zMTZ)qkrBX3^XvVy?C>UvPS#p#v=BP_KJN(kAd6HO_Y}{_2^0Y!jwuN%;VmQp?%!cS
z_nq=&*LWsYKM}no_+ys$$i@-q+FpSdeajXvb=D9ZFI7UR&#@sRRwF8fo0TE>w;4lK
zYgQ|j;nYREDcTI1k@<}E?aQ3XDctA&FQ$09$Bi*WlJcqyS^$qXD20(btREgCH6Z7Q
zE<Vz?{4p&ce~WqCf5HX1zT)UK|EW#RLW)4{q4+y!fDNqlUIHFqM+M;U@8FCw9V0U2
zj{KqyTFBqaQJy1hz;2iNMuGOP_f9hM0<s(o$iVU}%Mi=+XDMSsY6q<;39VUx4_qWU
z?3hIQQs$=VkQT8sP1xEvT4Tsd^3D%a?=Tc5e21|V`?kf;tAJe+X~9pyPUvkhlFFmV
z6EvCvE{Y*)W0l|JnZT8>7?B4PkP`Si&ME>vqSuP}z9q3<M3}$Nnm=SgFdCF-<ZVKN
zg@=R$?)}4{QuQy7>%)kCm@p{)b@n7{AIg^7?aiWJp3Gdne0gh<%LNWV5J}KSzEFf%
zjv^Qdb&tB!Y)1MAsSxJsxA~$#?gq~OD>#kXn<bC97W^xZm-d;((h2jV%jbAOwAROh
z9OAo{*OVjueLSg*%7)m9_9W3LiaQHq9`@>u<+Um75{mzdN&dGZmkh2l?%fc>{gnKF
zI-(-Eg!bS-{DAwrR}m)wiVeSlM1H$ukd=c`6=xN$QiK9}Q|iCzk^e$Q^4WvOqdMXL
zu24l?w<ax3`8*fVUu1n*`6tn^_#7n5M+E_Vi}$EZ%Os>0Iws|Tc@_c4_@6Ic!3=61
j0g1~z0cKCZo=kr@W9CZ5a;f|?+JW*L<@4o6`Gx-n?#Ui_

literal 0
HcmV?d00001

diff --git a/src/learning/learners.py b/src/learning/learners.py
new file mode 100644
index 0000000..e540018
--- /dev/null
+++ b/src/learning/learners.py
@@ -0,0 +1,646 @@
+import numpy as np
+import time
+from data.embeddings import WordEmbeddings, embedding_matrix, WCE_matrix
+from scipy.sparse import issparse, csr_matrix
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.model_selection import GridSearchCV
+from sklearn.model_selection import KFold
+# from sklearn.externals.joblib import Parallel, delayed
+from joblib import Parallel, delayed
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+
+def _sort_if_sparse(X):
+    if issparse(X) and not X.has_sorted_indices:
+        X.sort_indices()
+
+
+def _joblib_transform_multiling(transformer, lX, n_jobs=-1):
+    if n_jobs == 1:
+        return {lang:transformer(lX[lang]) for lang in lX.keys()}
+    else:
+        langs = list(lX.keys())
+        transformations = Parallel(n_jobs=n_jobs)(delayed(transformer)(lX[lang]) for lang in langs)
+        return {lang: transformations[i] for i, lang in enumerate(langs)}
+
+
+class TrivialRejector:
+    def fit(self, X, y):
+        self.cats = y.shape[1]
+        return self
+    def decision_function(self, X): return np.zeros((X.shape[0],self.cats))
+    def predict(self, X): return np.zeros((X.shape[0],self.cats))
+    def predict_proba(self, X): return np.zeros((X.shape[0],self.cats))
+    def best_params(self): return {}
+
+
+class FunnellingPolylingualClassifier:
+    """
+    This classifier projects each document d into a language-independent feature space where each dimension fi is the
+    decision score phi_l(d,ci) of an auxiliary classifier phi_l trained on category ci for documents in language l;
+    then trains one single classifier for all documents in this space, irrespective of their originary language
+    """
+    def __init__(self, first_tier_learner, meta_learner, first_tier_parameters=None, meta_parameters=None, folded_projections=1,
+                 calmode='cal', n_jobs=-1):
+        """
+        :param first_tier_learner: the learner used in the first-tier level
+        :param meta_learner: the learner used in the second-tier level
+        :param first_tier_parameters: parameters for the learner in the doc_projector
+        :param meta_parameters: parameters for the learner in the z-space
+        :param folded_projections: if 1 then the model trains the auxiliar classifiers with all training data and
+        :param calmode: 'cal' to calibrate the base classifiers, 'nocal' to use the decision_function instead, or
+        :param n_jobs: number of parallel threads
+        'sigmoid' to use the sigmoid of the decision_function
+        projects the data before training the final classifier; if greater than one, the training set is split in as
+        many folds as indicated, and the projected space is composed by concatenating each fold prediction based on
+        models trained on the remaining folds. This should increase the generality of the space to unseen data.
+        """
+        assert folded_projections>0, "positive number of folds expected"
+        assert calmode in ['cal','nocal','sigmoid'], 'unknown calmode'
+        assert calmode!='cal' or first_tier_learner.probability, 'calmode=cal requires the learner to have probability=True'
+
+        self.fist_tier_learner = first_tier_learner
+        self.meta_learner = meta_learner
+        self.fist_tier_parameters=first_tier_parameters
+        self.meta_parameters = meta_parameters
+        self.doc_projector = NaivePolylingualClassifier(self.fist_tier_learner, self.fist_tier_parameters, n_jobs=n_jobs)
+        self.doc_projector_bu = NaivePolylingualClassifier(self.fist_tier_learner, self.fist_tier_parameters, n_jobs=n_jobs)
+        self.folded_projections = folded_projections
+        self.n_jobs = n_jobs
+        self.calmode = calmode
+
+    def _projection(self, doc_projector, lX):
+        """
+        Decides the projection function to be applied; predict_proba if the base classifiers are calibrated or
+        decision_function if otherwise
+        :param doc_projector: the document projector (a NaivePolylingualClassifier)
+        :param lX: {lang:matrix} to train
+        :return: the projection, applied with predict_proba or decision_function
+        """
+        if self.calmode=='cal':
+            return doc_projector.predict_proba(lX)
+        else:
+            l_decision_scores = doc_projector.decision_function(lX)
+            if self.calmode=='sigmoid':
+                def sigmoid(x): return 1 / (1 + np.exp(-x))
+                for lang in l_decision_scores.keys():
+                    l_decision_scores[lang] = sigmoid(l_decision_scores[lang])
+            return l_decision_scores
+
+    def _get_zspace(self, lXtr, lYtr, lXproj=None, lYproj=None):
+        """
+        Produces the vector space of posterior probabilities (in case the first-tier is calibrated) or of
+        decision scores (if otherwise). This space is here named zspace.
+        :param lXtr: {lang:matrix} to train
+        :param lYtr: {lang:labels} to train
+        :param lXproj: {lang:matrix} to project (if None, then projects the lXtr)
+        :param lYproj: {lang:labels} to stack in the same order (if None, then lYtr will be stacked)
+        :return: the projection of lXproj documents into the Z-space defined by the confidence scores of language-specific
+        models trained on lXtr, and the lYproj labels stacked consistently
+        """
+        repair_empty_folds = True
+        if lXproj is None and lYproj is None:
+            lXproj, lYproj = lXtr, lYtr
+            repair_empty_folds = False
+
+        print('fitting the projectors... {}'.format(lXtr.keys()))
+        self.doc_projector.fit(lXtr, lYtr)
+
+        print('projecting the documents')
+        langs = list(lXtr.keys())
+        lZ = self._projection(self.doc_projector, lXproj)
+
+        # if repair_empty_folds: #empty folds are replaced by the posterior probabilities generated by the non-folded version
+        empty_categories = self.doc_projector.empty_categories
+        lZ_bu = self._projection(self.doc_projector_bu, lXproj)
+
+        for lang in langs:
+            repair = empty_categories[lang]
+            lZ[lang][:,repair] = lZ_bu[lang][:,repair]
+
+        Z = np.vstack([lZ[lang] for lang in langs])  # Z is the language independent space
+        zy = np.vstack([lYproj[lang] for lang in langs])
+        return Z, zy
+
+    def _get_zspace_folds(self, lX, ly):
+        self.doc_projector_bu.fit(lX, ly)
+
+        print('split of {} folds'.format(self.folded_projections))
+        skf = KFold(n_splits=self.folded_projections, shuffle=True)
+
+        Z, zy = [], []
+        lfold = {lang: list(skf.split(lX[lang], ly[lang])) for lang in lX.keys()}
+        for fold in range(self.folded_projections):
+            print('fitting the projectors ({}/{})...'.format(fold + 1, self.folded_projections))
+            lfoldXtr, lfoldYtr = {}, {}
+            lfoldXte, lfoldYte = {}, {}
+            for lang in lX.keys():
+                train, test = lfold[lang][fold]
+                lfoldXtr[lang] = lX[lang][train]
+                lfoldYtr[lang] = ly[lang][train]
+                lfoldXte[lang] = lX[lang][test]
+                lfoldYte[lang] = ly[lang][test]
+            Zfold, zYfold = self._get_zspace(lfoldXtr, lfoldYtr, lfoldXte, lfoldYte)
+            Z.append(Zfold)
+            zy.append(zYfold)
+        # compose the Z-space as the union of all folded predictions
+        Z = np.vstack(Z)
+        zy = np.vstack(zy)
+        # refit the document projector with all examples to have a more reliable projector for test data
+        self.doc_projector = self.doc_projector_bu
+        return Z, zy
+
+    def fit(self, lX, ly, lZ=None, lzy=None):
+        tinit = time.time()
+        Z, zy = self._get_zspace(lX, ly) if self.folded_projections == 1 else self._get_zspace_folds(lX, ly)
+
+        #experimental: adds the posterior probabilities (computed outside) to the meta-classifier
+        if lZ is not None and lzy is not None:
+            zlangs = list(lZ.keys())
+            Z = np.vstack((Z, *[lZ[l] for l in zlangs]))
+            zy = np.vstack((zy, *[lzy[l] for l in zlangs]))
+
+        print('fitting the Z-space of shape={}'.format(Z.shape))
+        self.model = MonolingualClassifier(base_learner=self.meta_learner, parameters=self.meta_parameters, n_jobs=self.n_jobs)
+        self.model.fit(Z, zy)
+        self.time = time.time() - tinit
+
+        return self
+
+    def predict(self, lX, lZ=None):
+        """
+        :param lX: a dictionary {language_label: X csr-matrix}
+        :param lZ: a dictionary {language_label: Z matrix}; if specified, concats this representation
+        :return: a dictionary of predictions
+        """
+        lZ_ = self._projection(self.doc_projector, lX)
+        if lZ is not None:
+            lZ_ = {**lZ_, **lZ}
+        return _joblib_transform_multiling(self.model.predict, lZ_, n_jobs=self.n_jobs)
+
+    def best_params(self):
+        params = self.doc_projector.best_params()
+        params['meta'] = self.model.best_params()
+        return params
+
+
+class NaivePolylingualClassifier:
+    """
+    Is a mere set of independet MonolingualClassifiers
+    """
+    def __init__(self, base_learner, parameters=None, n_jobs=-1):
+        self.base_learner = base_learner
+        self.parameters = parameters
+        self.model = None
+        self.n_jobs = n_jobs
+
+    def fit(self, lX, ly):
+        """
+        trains the independent monolingual classifiers
+        :param lX: a dictionary {language_label: X csr-matrix}
+        :param ly: a dictionary {language_label: y np.array}
+        :return: self
+        """
+        tinit = time.time()
+        assert set(lX.keys()) == set(ly.keys()), 'inconsistent language mappings in fit'
+        langs = list(lX.keys())
+        for lang in langs:
+            _sort_if_sparse(lX[lang])
+
+        # models = Parallel(n_jobs=self.n_jobs)\
+        #     (delayed(MonolingualClassifier(self.base_learner, parameters=self.parameters).fit)((lX[lang]),ly[lang]) for lang in langs)
+
+        models = [MonolingualClassifier(self.base_learner, parameters=self.parameters) for lang in langs]
+
+        for model, lang in zip(models, langs):
+            model.fit(lX[lang], ly[lang])
+
+        self.model = {lang: models[i] for i, lang in enumerate(langs)}
+        self.empty_categories = {lang:self.model[lang].empty_categories for lang in langs}
+        self.time = time.time() - tinit
+        return self
+
+    def decision_function(self, lX):
+        """
+        :param lX: a dictionary {language_label: X csr-matrix}
+        :return: a dictionary of classification scores for each class
+        """
+        assert self.model is not None, 'predict called before fit'
+        assert set(lX.keys()).issubset(set(self.model.keys())), 'unknown languages requested in decision function'
+        langs=list(lX.keys())
+        scores = Parallel(n_jobs=self.n_jobs)(delayed(self.model[lang].decision_function)(lX[lang]) for lang in langs)
+        return {lang:scores[i] for i,lang in enumerate(langs)}
+
+    def predict_proba(self, lX):
+        """
+        :param lX: a dictionary {language_label: X csr-matrix}
+        :return: a dictionary of probabilities that each document belongs to each class
+        """
+        assert self.model is not None, 'predict called before fit'
+        assert set(lX.keys()).issubset(set(self.model.keys())), 'unknown languages requested in decision function'
+        langs=list(lX.keys())
+        scores = Parallel(n_jobs=self.n_jobs, max_nbytes=None)(delayed(self.model[lang].predict_proba)(lX[lang]) for lang in langs)
+        return {lang:scores[i] for i,lang in enumerate(langs)}
+
+    def predict(self, lX):
+        """
+        :param lX: a dictionary {language_label: X csr-matrix}
+        :return: a dictionary of predictions
+        """
+        assert self.model is not None, 'predict called before fit'
+        assert set(lX.keys()).issubset(set(self.model.keys())), 'unknown languages requested in predict'
+        if self.n_jobs == 1:
+            return {lang:self.model[lang].predict(lX[lang]) for lang in lX.keys()}
+        else:
+            langs = list(lX.keys())
+            scores = Parallel(n_jobs=self.n_jobs)(delayed(self.model[lang].predict)(lX[lang]) for lang in langs)
+            return {lang: scores[i] for i, lang in enumerate(langs)}
+
+    def best_params(self):
+        return {l:model.best_params() for l,model in self.model.items()}
+
+
+class MonolingualClassifier:
+
+    def __init__(self, base_learner, parameters=None, n_jobs=-1):
+        self.learner = base_learner
+        self.parameters = parameters
+        self.model = None
+        self.n_jobs = n_jobs
+        self.best_params_ = None
+
+    def fit(self, X, y):
+        if X.shape[0] == 0:
+            print('Warning: X has 0 elements, a trivial rejector will be created')
+            self.model = TrivialRejector().fit(X,y)
+            self.empty_categories = np.arange(y.shape[1])
+            return self
+
+        tinit = time.time()
+        _sort_if_sparse(X)
+        self.empty_categories = np.argwhere(np.sum(y, axis=0)==0).flatten()
+
+        # multi-class format
+        if len(y.shape) == 2:
+            if self.parameters is not None:
+                self.parameters = [{'estimator__' + key: params[key] for key in params.keys()}
+                                   for params in self.parameters]
+            self.model = OneVsRestClassifier(self.learner, n_jobs=self.n_jobs)
+        else:
+            self.model = self.learner
+            raise NotImplementedError('not working as a base-classifier for funneling if there are gaps in the labels across languages')
+
+        # parameter optimization?
+        if self.parameters:
+            print('debug: optimizing parameters:', self.parameters)
+            self.model = GridSearchCV(self.model, param_grid=self.parameters, refit=True, cv=5, n_jobs=self.n_jobs,
+                                      error_score=0, verbose=10)
+
+        print('fitting:', self.model)
+        self.model.fit(X, y)
+        if isinstance(self.model, GridSearchCV):
+            self.best_params_ = self.model.best_params_
+            print('best parameters: ', self.best_params_)
+        self.time=time.time()-tinit
+        return self
+
+    def decision_function(self, X):
+        assert self.model is not None, 'predict called before fit'
+        _sort_if_sparse(X)
+        return self.model.decision_function(X)
+
+    def predict_proba(self, X):
+        assert self.model is not None, 'predict called before fit'
+        assert hasattr(self.model, 'predict_proba'), 'the probability predictions are not enabled in this model'
+        _sort_if_sparse(X)
+        return self.model.predict_proba(X)
+
+    def predict(self, X):
+        assert self.model is not None, 'predict called before fit'
+        _sort_if_sparse(X)
+        return self.model.predict(X)
+
+    def best_params(self):
+        return self.best_params_
+
+
+class PolylingualEmbeddingsClassifier:
+    """
+    This classifier creates document embeddings by a tfidf weighted average of polylingual embeddings from the article
+    @article{conneau2017word,
+      title={Word translation without parallel data},
+      author={Conneau, Alexis and Lample, Guillaume and Ranzato, Marc'Aurelio and Denoyer, Ludovic and J{\'e}gou, Herv{\'e}},
+      journal={arXiv preprint arXiv:1710.04087},
+      year={2017}
+    }
+    url: https://github.com/facebookresearch/MUSE
+    """
+    def __init__(self, wordembeddings_path, learner, c_parameters=None, n_jobs=-1):
+        """
+        :param wordembeddings_path: the path to the directory containing the polylingual embeddings
+        :param learner: the learner
+        :param c_parameters: parameters for learner
+        :param n_jobs: the number of concurrent threads
+        """
+        self.wordembeddings_path = wordembeddings_path
+        self.learner = learner
+        self.c_parameters=c_parameters
+        self.n_jobs = n_jobs
+        self.lang_tfidf = {}
+        self.model = None
+
+    def fit_vectorizers(self, lX):
+        for lang in lX.keys():
+            if lang not in self.lang_tfidf:
+                tfidf = TfidfVectorizer(sublinear_tf=True, use_idf=True)  # text is already processed
+                docs = lX[lang]
+                tfidf.fit(docs)
+                self.lang_tfidf[lang] = tfidf
+
+    def embed(self, docs, lang):
+        assert lang in self.lang_tfidf, 'unknown language'
+        tfidf_vectorizer = self.lang_tfidf[lang]
+        V = tfidf_vectorizer.vocabulary_
+        Xweights = tfidf_vectorizer.transform(docs)
+
+        print('loading word embeddings for ' + lang)
+        we = WordEmbeddings.load(self.wordembeddings_path, lang)
+
+        nD = len(docs)
+        doc_vecs = np.zeros((nD, we.dim()))
+
+        for i, doc in enumerate(docs):
+            print('\r\tcomplete {:.3f}%'.format(100 * (i + 1) / nD), end='')
+            # averaging with tfidf (summing each word only once, since the frequency is already controlled)
+            for w in set(doc.split()):
+                if w in we and w in V:
+                    doc_vecs[i] += (we[w] * Xweights[i, V[w]])
+            # works much worse with idf; works much worse with document l2-normalization
+        print()
+
+        return doc_vecs
+
+    def fit(self, lX, ly):
+        """
+        :param lX: a dictionary {language_label: [list of preprocessed documents]}
+        :param ly: a dictionary {language_label: ndarray of shape (ndocs, ncats) binary labels}
+        :return: self
+        """
+        tinit = time.time()
+        langs = list(lX.keys())
+        WEtr, Ytr = [], []
+        self.fit_vectorizers(lX) # if already fit, does nothing
+        _lX = dict()
+        for lang in langs:
+            _lX[lang] = self.lang_tfidf[lang].transform(lX[lang])
+            WEtr.append(self.embed(lX[lang], lang))
+            Ytr.append(ly[lang])
+
+        # TODO @Andrea --> here embeddings should be stacked horizontally!
+        WEtr = np.vstack(WEtr)
+        Ytr = np.vstack(Ytr)
+        self.embed_time = time.time() - tinit
+
+        print('fitting the WE-space of shape={}'.format(WEtr.shape))
+        self.model = MonolingualClassifier(base_learner=self.learner, parameters=self.c_parameters, n_jobs=self.n_jobs)
+        self.model.fit(_lX['da'], ly['da'])
+        self.time = time.time() - tinit
+        return self
+
+    def predict(self, lX):
+        """
+        :param lX: a dictionary {language_label: [list of preprocessed documents]}
+        """
+        assert self.model is not None, 'predict called before fit'
+        langs = list(lX.keys())
+        lWEte = {lang:self.embed(lX[lang], lang) for lang in langs} # parallelizing this may consume too much memory
+        return _joblib_transform_multiling(self.model.predict, lWEte, n_jobs=self.n_jobs)
+
+    def predict_proba(self, lX):
+        """
+        :param lX: a dictionary {language_label: [list of preprocessed documents]}
+        """
+        assert self.model is not None, 'predict called before fit'
+        langs = list(lX.keys())
+        # lWEte = {lang:self.embed(lX[lang], lang) for lang in langs} # parallelizing this may consume too much memory
+        return _joblib_transform_multiling(self.model.predict_proba, self.lang_tfidf['da'], n_jobs=self.n_jobs)
+
+    def best_params(self):
+        return self.model.best_params()
+
+
+class FunnellingEmbeddingPolylingualClassifier:
+    """ Simulated: this setting is merely for testing purposes, and is not realistic. We here assume to have a tfidf
+    vectorizer for the out-of-scope languages (which is not fair)."""
+    def __init__(self, first_tier_learner, embed_learner, meta_learner, wordembeddings_path, training_languages,
+                 first_tier_parameters = None, embed_parameters = None, meta_parameters = None, n_jobs=-1):
+
+        assert first_tier_learner.probability==True and embed_learner.probability==True, \
+            'both the first-tier classifier and the polyembedding classifier shoud allow calibration'
+
+        self.training_languages = training_languages
+
+        self.PLE = PolylingualEmbeddingsClassifier(wordembeddings_path, embed_learner,
+                                                   c_parameters=embed_parameters, n_jobs=n_jobs)
+
+        self.Funnelling = FunnellingPolylingualClassifier(first_tier_learner, meta_learner,
+                                                          first_tier_parameters=first_tier_parameters,
+                                                          meta_parameters=meta_parameters, n_jobs=n_jobs)
+        self.n_jobs = n_jobs
+
+    def vectorize(self, lX):
+        return {l:self.PLE.lang_tfidf[l].transform(lX[l]) for l in lX.keys()}
+
+    def fit(self, lX, ly):
+        """
+        :param lX: a dictionary {language_label: [list of preprocessed documents]}
+        :param ly: a dictionary {language_label: ndarray of shape (ndocs, ncats) binary labels}
+        :return:
+        """
+        self.PLE.fit_vectorizers(lX)
+        tinit = time.time()
+        lX = {l: lX[l] for l in lX.keys() if l in self.training_languages}
+        ly = {l: ly[l] for l in lX.keys() if l in self.training_languages}
+        self.PLE.fit(lX, ly)
+        lZ = self.PLE.predict_proba(lX)
+        self.Funnelling.fit(self.vectorize(lX),ly,lZ,ly)
+        self.time = time.time() - tinit
+        return self
+
+    def predict(self, lX):
+        """
+        :param lX: a dictionary {language_label: [list of preprocessed documents]}
+        """
+        lXin = {l: lX[l] for l in lX.keys() if l in self.training_languages}
+        lXout = {l: lX[l] for l in lX.keys() if l not in self.training_languages}
+
+        lZ = self.PLE.predict_proba(lXout)
+
+        return self.Funnelling.predict(self.vectorize(lXin), lZ)
+
+
+    def best_params(self):
+        return {'PLE':self.PLE.best_params(), 'Funnelling':self.Funnelling.best_params()}
+
+
+class AndreaCLF(FunnellingPolylingualClassifier):
+    def __init__(self,
+                 we_path,
+                 config,
+                 first_tier_learner,
+                 meta_learner,
+                 first_tier_parameters=None,
+                 meta_parameters=None,
+                 folded_projections=1,
+                 calmode='cal', n_jobs=-1):
+
+        super().__init__(first_tier_learner,
+                         meta_learner,
+                         first_tier_parameters,
+                         meta_parameters,
+                         folded_projections,
+                         calmode,
+                         n_jobs)
+
+        self.we_path = we_path
+        self.config = config
+        self.lang_word2idx = dict()
+        self.languages = []
+        self.lang_tfidf = {}
+        self.word_embeddings = {}
+        self.supervised_embeddings = {}
+
+    def vectorize(self, lX, prediction=False):
+        langs = list(lX.keys())
+        print(f'# tfidf-vectorizing docs')
+        if prediction:
+            for lang in langs:
+                assert lang in self.lang_tfidf.keys(), 'no tf-idf for given language'
+                tfidf_vectorizer = self.lang_tfidf[lang]
+                lX[lang] = tfidf_vectorizer.transform(lX[lang])
+            return self
+
+        for lang in langs:
+            tfidf_vectorizer = TfidfVectorizer(sublinear_tf=True, use_idf=True)
+            self.languages.append(lang)
+            tfidf_vectorizer.fit(lX[lang])
+            lX[lang] = tfidf_vectorizer.transform(lX[lang])
+            _sort_if_sparse(lX[lang])
+            self.lang_word2idx[lang] = tfidf_vectorizer.vocabulary_
+            self.lang_tfidf[lang] = tfidf_vectorizer # utile in fase di testing
+        return self
+
+    # @override std class method
+    def _get_zspace(self, lXtr, lYtr):
+        print('\nfitting the projectors... {}'.format(list(lXtr.keys())))
+        self.doc_projector.fit(lXtr, lYtr)
+
+        print('\nprojecting the documents')
+        lZ = self._projection(self.doc_projector, lXtr)
+
+        return lZ, lYtr
+
+    def embed(self, lX, ly, unsupervised=False, supervised=False, prediction=False):
+        """
+        build embedding matrix for given language and returns its weighted sum wrt tf-idf score
+        """
+        _r = dict()
+        languages = list(lX.keys())
+
+        if prediction:
+            for lang in languages:
+                if unsupervised:    # If unsupervised embeddings ...
+                    M = self.word_embeddings[lang]
+                    if supervised:  # and also unsupervised --> get both (M) and (S) weighted sum matrices and hstack them
+                        S = self.supervised_embeddings[lang]
+                        _r[lang] = np.hstack((lX[lang].dot(M), lX[lang].dot(S)))
+                        continue
+                    _r[lang] = lX[lang].dot(M)  # if not supervised --> just get weighted sum of unsupervised (M) embeddings
+                else:   # If not unsupervised --> get (S) matrix and its weighted sum 
+                    S = self.supervised_embeddings[lang]
+                    _r[lang] = lX[lang].dot(S)
+            return _r
+
+        if unsupervised:
+            for lang in languages:
+                # print('Test building embedding matrix FastTextMuse ...')
+                _, M = embedding_matrix(self.we_path, self.lang_word2idx[lang], lang)
+                self.word_embeddings[lang] = M
+                _r[lang] = lX[lang].dot(M)
+
+        if supervised:
+            for lang in languages:
+                S = WCE_matrix(lX, ly, lang)
+                S = np.squeeze(np.asarray(S))   # casting to ndarray to better visualize S while debugging
+                self.supervised_embeddings[lang] = S
+                if unsupervised:
+                    _r[lang] = np.hstack((_r[lang], lX[lang].dot(S)))
+                else:
+                    _r[lang] = lX[lang].dot(S)
+        return _r
+
+    # @override std class method
+    def fit(self, lX, ly):
+        tinit = time.time()
+        print('Vectorizing documents...')
+        self.vectorize(lX)
+
+        for lang in self.languages:
+            print(lX[lang].shape)
+
+        Z, zy = self._get_zspace(lX, ly)
+
+        # Z vectors is concatenated with doc's embedding weighted sum
+        Z_embedded = dict()
+        l_weighted_em = self.embed(lX, ly,
+                                   unsupervised=self.config['unsupervised'],
+                                   supervised=self.config['supervised'])
+
+        if self.config['supervised'] or self.config['unsupervised']:
+            for lang in list(lX.keys()):
+                Z_embedded[lang] = np.hstack((Z[lang], l_weighted_em[lang]))
+            Z = Z_embedded
+            del Z_embedded
+
+        # stacking Z_embedded space vertically
+        # _vertical_Z = np.vstack([Z_embedded[lang] for lang in self.languages])
+        # _vertical_Zy = np.vstack([zy[lang] for lang in self.languages])
+        _vertical_Z = np.vstack([Z[lang] for lang in self.languages])
+        _vertical_Zy = np.vstack([zy[lang] for lang in self.languages])
+
+        # zlangs = list(Z_embedded.keys())  # creo lista con embedding e poi faccio vstack su lista
+        # for i, lang in enumerate(zlangs):
+        #     if i == 0:
+        #         _vertical_Z = Z_embedded[lang]
+        #         _vertical_Zy = zy[lang]
+        #     else:
+        #         _vertical_Z = np.vstack((_vertical_Z, Z_embedded[lang]))
+        #         _vertical_Zy = np.vstack((_vertical_Zy, zy[lang]))
+
+        print('fitting the Z-space of shape={}'.format(_vertical_Z.shape))
+        self.model = MonolingualClassifier(base_learner=self.meta_learner, parameters=self.meta_parameters,
+                                           n_jobs=self.n_jobs)
+        self.model.fit(_vertical_Z, _vertical_Zy)
+        self.time = time.time() - tinit
+        print(f'\nTotal training time elapsed: {round((self.time/60), 2)} min')
+
+    def predict(self, lX, ly):
+        print('Vectorizing documents')
+        self.vectorize(lX, prediction=True)
+        lZ = self._projection(self.doc_projector, lX)
+
+        if self.config['supervised'] or self.config['unsupervised']:
+            l_weighted_em = self.embed(lX, ly,
+                                       unsupervised=self.config['unsupervised'],
+                                       supervised=self.config['supervised'],
+                                       prediction=True)
+            Z_embedded = dict()
+            for lang in lX.keys():
+                Z_embedded[lang] = np.hstack((lZ[lang], l_weighted_em[lang]))
+                print(Z_embedded[lang].shape)
+            
+            return _joblib_transform_multiling(self.model.predict, Z_embedded, n_jobs=self.n_jobs)
+        
+        for lang in lZ.keys():
+            print(lZ[lang].shape)
+        return _joblib_transform_multiling(self.model.predict, lZ, n_jobs=self.n_jobs)
diff --git a/src/results/results.csv b/src/results/results.csv
new file mode 100644
index 0000000..783225c
--- /dev/null
+++ b/src/results/results.csv
@@ -0,0 +1,7 @@
+id	method	learner	embed	optimp	dataset	binary	languages	time	lang	macrof1	microf1	macrok	microk	notes
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_M	PolyEmbed_andrea	svm	M	False	test_datasetname	not_binary	not_ablation	55.56810355186462	da	0.7933333333333333	0.0	0.7933333333333333	0.0	nope
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_M	PolyEmbed_andrea	svm	M	False	test_datasetname	not_binary	not_ablation	55.56810355186462	en	0.7866666666666666	0.0	0.7927111111111111	-0.0003376325207643527	nope
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_M	PolyEmbed_andrea	svm	M	False	test_datasetname	not_binary	not_ablation	55.56810355186462	fr	0.7866666666666666	0.0	0.7930666666666667	-0.0001350530083057411	nope
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_None	PolyEmbed_andrea	svm	None	False	test_datasetname	not_binary	not_ablation	24.031760931015015	da	0.7933333333333333	0.0	0.7933333333333333	0.0	nope
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_None	PolyEmbed_andrea	svm	None	False	test_datasetname	not_binary	not_ablation	24.031760931015015	en	0.7933333333333333	0.0	0.7931111111111111	-0.00013505300830574107	nope
+jrc_doclist_1958-2005vs2006_all_top300_noparallel_processed_run0.picklePolyEmbedd_andrea_None	PolyEmbed_andrea	svm	None	False	test_datasetname	not_binary	not_ablation	24.031760931015015	fr	0.7933333333333333	0.0	0.7933333333333333	0.0	nope
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/transformers/clesa.py b/src/transformers/clesa.py
new file mode 100644
index 0000000..da17393
--- /dev/null
+++ b/src/transformers/clesa.py
@@ -0,0 +1,110 @@
+import numpy as np
+import sklearn
+# from sklearn.externals.joblib import Parallel, delayed
+from joblib import Parallel, delayed
+
+class ESA(object):
+    """
+    Implementation of Explicit Sematic Analysis (ESA) in its mono-lingual version, as a transformer
+    """
+    supported_similarity = ['dot', 'cosine']
+
+    def __init__(self, similarity='dot', centered=False, post=None):
+        """
+        :param similarity: the similarity measure between documents to be used
+        :param centered: set to True to subtract the expected similarity due to randomness (experimental)
+        :param post: any valid sklearn normalization method to be applied to the resulting doc embeddings, or None (default)
+        """
+        assert similarity in self.supported_similarity, ("Similarity method %s is not supported" % similarity)
+        self.similarity = similarity
+        self.centered = centered
+        self.post_processing = post
+        self.W = None
+
+    def fit(self, W):
+        """
+        :param W: doc-by-term already processed matrix of wikipedia documents
+        :return: self
+        """
+        self.W = W
+        return self
+
+    def transform(self, X):
+        """
+        :param X: doc-by-term matrix that is to be transformed into the ESA space.
+        :return: the matrix X transformed into the ESA space in numpy format
+        """
+        assert self.W is not None, 'transform method called before fit'
+
+        W = self.W
+        assert X.shape[1] == W.shape[1], ('the feature spaces for X=%s and W=%s do not agree' % (str(X.shape), str(W.shape)))
+
+        if self.similarity in ['dot', 'cosine']:
+            if self.similarity == 'cosine':
+                X = sklearn.preprocessing.normalize(X, norm='l2', axis=1, copy=True)
+                W = sklearn.preprocessing.normalize(W, norm='l2', axis=1, copy=True)
+
+            esa = (X.dot(W.T)).toarray()
+            if self.centered:
+                pX = (X > 0).sum(1) / float(X.shape[1])
+                pW = (W > 0).sum(1) / float(W.shape[1])
+                pXpW = np.sqrt(pX.dot(pW.transpose()))
+                esa = esa - pXpW
+
+            if self.post_processing:
+                esa = sklearn.preprocessing.normalize(esa, norm=self.post_processing, axis=1, copy=True)
+
+            return esa
+
+    def fit_transform(self, W, X, Y=None):
+        self.fit(W)
+        return self.transform(X, Y)
+
+    def dimensionality(self):
+        return self.W.shape[0]
+
+
+
+class CLESA(ESA):
+    """
+    Implementation of Cross-Lingual Explicit Sematic Analysis (ESA) as a transformer
+    """
+    
+    def __init__(self, similarity='dot', centered=False, post=False, n_jobs=-1):
+        super(CLESA, self).__init__(similarity, centered, post)
+        self.lESA = None
+        self.langs = None
+        self.n_jobs = n_jobs
+
+    def fit(self, lW):
+        """
+        :param lW: a dictionary of {language: doc-by-term wiki matrix}
+        :return: self
+        """
+        assert len(np.unique([W.shape[0] for W in lW.values()])) == 1, "inconsistent dimensions across languages"
+
+        self.dimensions = list(lW.values())[0].shape[0]
+        self.langs = list(lW.keys())
+        self.lESA = {lang:ESA(self.similarity, self.centered, self.post_processing).fit(lW[lang]) for lang in self.langs}
+        return self
+
+    def transform(self, lX):
+        """
+        :param lX: dictionary of {language : doc-by-term matrix} that is to be transformed into the CL-ESA space
+        :return: a dictionary {language : doc-by-dim matrix} containing the matrix-transformed versions
+        """
+        assert self.lESA is not None, 'transform method called before fit'
+        assert set(lX.keys()).issubset(set(self.langs)), 'languages in lX are not scope'
+        langs = list(lX.keys())
+        trans = Parallel(n_jobs=self.n_jobs)(delayed(self.lESA[lang].transform)(lX[lang]) for lang in langs)
+        return {lang:trans[i] for i,lang in enumerate(langs)}
+
+    def fit_transform(self, lW, lX):
+        return self.fit(lW).transform(lX)
+
+    def languages(self):
+        return list(self.lESA.keys())
+
+
+
+
diff --git a/src/transformers/dci.py b/src/transformers/dci.py
new file mode 100644
index 0000000..6e84ed9
--- /dev/null
+++ b/src/transformers/dci.py
@@ -0,0 +1,154 @@
+import numpy as np
+from sklearn.preprocessing import normalize
+from scipy.sparse import csr_matrix, issparse
+from scipy.spatial.distance import cosine
+import operator
+import functools
+import math, sys
+# from sklearn.externals.joblib import Parallel, delayed
+from joblib import Parallel, delayed
+
+
+class DistributionalCorrespondenceIndexing:
+
+    prob_dcf = ['linear', 'pmi']
+    vect_dcf = ['cosine']
+    valid_dcf = prob_dcf + vect_dcf
+    valid_post = ['normal', 'l2', None]
+
+    def __init__(self, dcf='cosine', post='normal', n_jobs=-1):
+        """
+        :param dcf: a distributional correspondence function name (e.g., 'cosine') or a callable f(u,v) which measures
+                the distribucional correspondence between vectors u and v
+        :param post: post-processing function to apply to document embeddings. Default is to standardize it into a
+                normal distribution; other functions allowed are 'l2' or None
+        """
+        if post not in self.valid_post:
+            raise ValueError("unknown post processing function; valid ones are [%s]" % ', '.join(self.valid_post))
+
+        if isinstance(dcf, str):
+            if dcf not in self.valid_dcf:
+                raise ValueError("unknown dcf; use any in [%s]" % ', '.join(self.valid_dcf))
+            self.dcf = getattr(DistributionalCorrespondenceIndexing, dcf)
+        elif hasattr(dcf, '__call__'):
+            self.dcf = dcf
+        else:
+            raise ValueError('param dcf should either be a valid dcf name in [%s] or a callable comparing two vectors')
+        #self.dcf = lambda u,v:dcf(u,v)
+        self.post = post
+        self.domains = None
+        self.dFP = None
+        self.n_jobs = n_jobs
+
+    def fit(self, dU, dP):
+        """
+        :param dU: a dictionary of {domain:dsm_matrix}, where dsm is a document-by-term matrix representing the
+                distributional semantic model for a specific domain
+        :param dP: a dictionary {domain:pivot_matrix} where domain is a string representing each domain,
+                and pivot_matrix has shape (d,p) with d the dimensionality of the distributional space, and p the
+                number of pivots
+        :return: self
+        """
+        self.domains = list(dP.keys())
+        assert len(np.unique([P.shape[1] for P in dP.values()]))==1, "inconsistent number of pivots across domains"
+        assert set(dU.keys())==set(self.domains), "inconsistent domains in dU and dP"
+        assert not [1 for d in self.domains if dU[d].shape[0]!=dP[d].shape[0]], \
+            "inconsistent dimensions between distributional and pivot spaces"
+        self.dimensions = list(dP.values())[0].shape[1]
+        # embed the feature space from each domain using the pivots of that domain
+        #self.dFP = {d:self.dcf_dist(dU[d].transpose(), dP[d].transpose()) for d in self.domains}
+        transformations = Parallel(n_jobs=self.n_jobs)(delayed(self.dcf_dist)(dU[d].transpose(),dP[d].transpose()) for d in self.domains)
+        self.dFP = {d: transformations[i] for i, d in enumerate(self.domains)}
+
+    def _dom_transform(self, X, FP):
+        _X = X.dot(FP)
+        if self.post == 'l2':
+            _X = normalize(_X, norm='l2', axis=1)
+        elif self.post == 'normal':
+            std = np.clip(np.std(_X, axis=0), 1e-5, None)
+            _X = (_X - np.mean(_X, axis=0)) / std
+        return _X
+
+    # dX is a dictionary of {domain:dsm}, where dsm (distributional semantic model) is, e.g., a document-by-term csr_matrix
+    def transform(self, dX):
+        assert self.dFP is not None, 'transform method called before fit'
+        assert set(dX.keys()).issubset(self.domains), 'domains in dX are not scope'
+        domains = list(dX.keys())
+        transformations = Parallel(n_jobs=self.n_jobs)(delayed(self._dom_transform)(dX[d], self.dFP[d]) for d in domains)
+        return {d: transformations[i] for i, d in enumerate(domains)}
+
+    def fit_transform(self, dU, dP, dX):
+        return self.fit(dU, dP).transform(dX)
+
+    def _prevalence(self, v):
+        if issparse(v):
+            return float(v.nnz) / functools.reduce(operator.mul, v.shape, 1) #this works for arrays of any rank
+        elif isinstance(v, np.ndarray):
+            return float(v[v>0].size) / v.size
+
+    def linear(self, u, v, D):
+        tp, fp, fn, tn = self._get_4cellcounters(u, v, D)
+        den1=tp+fn
+        den2=tn+fp
+        tpr = (tp*1./den1) if den1!=0 else 0.
+        tnr = (tn*1./den2) if den2!=0 else 0.
+        return tpr + tnr - 1
+
+    def pmi(self, u, v, D):
+        tp, fp, fn, tn = self._get_4cellcounters(u, v, D)
+
+        Pxy = tp * 1. / D
+        Pxny = fp * 1. / D
+        Pnxy = fn * 1. / D
+        Px = Pxy + Pxny
+        Py = Pxy + Pnxy
+
+        if (Px == 0 or Py == 0 or Pxy == 0):
+            return 0.0
+
+        score =  math.log2(Pxy / (Px * Py))
+        if np.isnan(score) or np.isinf(score):
+            print('NAN')
+            sys.exit()
+        return score
+
+    def cosine(self, u, v):
+        pu = self._prevalence(u)
+        pv = self._prevalence(v)
+        return cosine(u, v) - np.sqrt(pu * pv)
+
+    def _get_4cellcounters(self, u, v, D):
+        """
+        :param u: a set of indexes with a non-zero value
+        :param v: a set of indexes with a non-zero value
+        :param D: the number of events (i.e., all posible indexes)
+        :return: the 4-cell contingency values tp, fp, fn, tn)
+        """
+        common=u.intersection(v)
+        tp = len(common)
+        fp = len(u) - len(common)
+        fn = len(v) - len(common)
+        tn = D - (tp + fp + fn)
+        return tp, fp, fn, tn
+
+    def dcf_dist(self, U, V):
+        nU,D = U.shape
+        nV = V.shape[0]
+        if issparse(U): U = U.toarray()
+        if issparse(V): V = V.toarray()
+
+        dists = np.zeros((nU, nV))
+        if self.dcf.__name__ in self.prob_dcf:
+            def hits_index(v):
+                return set(np.argwhere(v>0).reshape(-1).tolist())
+            Vhits = {i:hits_index(V[i]) for i in range(nV)}
+            for i in range(nU):
+                Ui_hits = hits_index(U[i])
+                for j in range(nV):
+                    dists[i, j] = self.dcf(self, Ui_hits, Vhits[j], D)
+        else:
+            for i in range(nU):
+                for j in range(nV):
+                    dists[i, j] = self.dcf(self, U[i], V[j])
+        return dists
+
diff --git a/src/transformers/riboc.py b/src/transformers/riboc.py
new file mode 100644
index 0000000..7dfbc42
--- /dev/null
+++ b/src/transformers/riboc.py
@@ -0,0 +1,53 @@
+import math
+import numpy as np
+from scipy.sparse import csr_matrix, issparse
+
+class RandomIndexingBoC(object):
+
+    def __init__(self, latent_dimensions, non_zeros=2):
+        self.latent_dimensions = latent_dimensions
+        self.k = non_zeros
+        self.ri_dict = None
+
+    def fit_transform(self, X):
+        return self.fit(X).transform(X)
+
+    def fit(self, X):
+        nF = X.shape[1]
+        nL = self.latent_dimensions
+        format = 'csr' if issparse(X) else 'np'
+        self.ri_dict = _create_random_index_dictionary(shape=(nF, nL), k=self.k, normalized=True, format=format)
+        return self
+
+    def transform(self, X):
+        assert X.shape[1] == self.ri_dict.shape[0], 'feature space is inconsistent with the RI dictionary'
+        if self.ri_dict is None:
+            raise ValueError("Error: transform method called before fit.")
+        P = X.dot(self.ri_dict)
+        if issparse(P):
+            P.sort_indices()
+        return P
+
+
+def _create_random_index_dictionary(shape, k, normalized=False, format='csr', positive=False):
+    assert format in ['csr', 'np'], 'Format should be in "[csr, np]"'
+    nF, latent_dimensions = shape
+    print("Creating the random index dictionary for |V|={} with {} dimensions".format(nF,latent_dimensions))
+    val = 1.0 if not normalized else 1.0/math.sqrt(k)
+    #ri_dict = csr_matrix((nF, latent_dimensions))  if format == 'csr' else np.zeros((nF, latent_dimensions))
+    ri_dict = np.zeros((nF, latent_dimensions))
+
+    #TODO: optimize
+    for t in range(nF):
+        dims = np.zeros(k, dtype=np.int32)
+        dims[0] = t % latent_dimensions #the first dimension is choosen in a round-robin manner (prevents gaps)
+        dims[1:] = np.random.choice(latent_dimensions, size=k-1, replace=False)
+        values = (np.random.randint(0,2, size=k)*2.0-1.0) * val if not positive else np.array([+val]*k)
+        ri_dict[t,dims]=values
+        print("\rprogress [%.2f%% complete]" % (t * 100.0 / nF), end='')
+    print('\nDone')
+
+    if format=='csr':
+        ri_dict = csr_matrix(ri_dict)
+    return ri_dict
+
diff --git a/src/util/__pycache__/evaluation.cpython-37.pyc b/src/util/__pycache__/evaluation.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c9c1d30d645d0b075eff16950a70c41a0fcab6d7
GIT binary patch
literal 4195
zcmd5<TXP&o74Gi2?Ck1lNs;ASSb`nAsUTSx0+q-jfuI7MsHn0k30rY(jkjl|k!EMs
zJ-x;&W?sB5EGYcMACMmT1N;u2=Yba<DE|e&z;}9fSC*}!s0t5ERi8fH)6;$C+`iL4
zotddIJpcUD-|zozma%`(VEkBMK1NG_4<ec55$p0M?~112G!~JyL%kK*U8m`oIXiNj
z70lU@*Yu<#-KVTsl@;lsuLV_E?Fd<uQ%`wwO0LKmISX!DUXt_j0=OA@SuV&$aI^A?
zT#|2ro0H4(BEFuN_C40P`U0}BhA6yW`^t}^ASx;{i2Oky8@%A}fq0$uR@zQfu-z84
z7!UO>+RxC^4v1zM+vh`$o>Ol#iya76mmG96wmFM0Yo^7qm9Z1Nm*T)yo7x_-Cp^Bc
zoebX%g?2}MWz>6R-+`o(L0W731;=SG==?^tu%aL?oOI9c1%-%vh1-wAhy9>f+)nf_
zyS*sr2C)w0&y`A4v(ok3D%n~G6PAo?X6`m!GjSJ6;DbU8T7O|`4hOvaXl*a)25WvS
zRp76skHV<m>m_SCNW)gAAICuy#=C2&YOnQm7_9}5{HX8iFo{=tgJR)y*y;vah3&M0
zO;b`E;~#KOs0-l!4^*m1Bt^nCs>&n2N`(AaO%a&}Y0jkGB++|gC60$9RZhS1_HJ`L
zqL)ysnxS0~4W(X<a@9PB#l<w~=+>K>q7Pox)B?sY{)MM)gQLemBc5U^eFUNrPscp7
z$h$bpEO?kux1RB5;+fU4q$TaA-jO&KC+t%;<oB4i%F&SLwzjd}dCn)RU0HeRWj3uG
z@;|U=tiz>8JHS)zAuo3@9$DQ!Tt1b*elu?EClAs)zsnm&t!;8wdd3m62tKWYgov2?
zD>*+BM?B+~*byS&5*vz)|5-eQzn5AfKZIY3)mw!f`SEVUDk{;S)l-2qj8<^>%E-S?
z75lQbA;Y$AC*9uWGM!i+e|U8yYOuNyC2c=SH&;&=BGlbd{4-LMrf<_8*Fhlj)f`cN
zB!>K~++Q%x9txuqzTM!83<J|Ip;K=WZz+;QT_mzXg#OLA!OQB#ZV(5LdusC%=8Uj5
zJG^p^u;X<T3F#Q}1Ebf=V2oZDz7{#pMXA$FyrEIc$W`#L6pa@Sot2)7q(~6jZlt+}
zTR1%x#=3AjiR${gun&Sk3ZsOnR;1mc8pQo>pnM%PoboTt|C(Q=oeGBxSf0_a)iq4L
zt`QRx-uSo8FH!6<KFd9RSxoafS7e2qN!Fl9rN}jjScBPRxp@4Gj2$y_^!pn*5^iQ)
zVqe?2ojbXkS2BSxu`?%gv&yc0Lczk-tQHc%L-QSL8Ey|_&VGp?*|)WI>}28uDO+04
zMebpB53B9W8v?d64{6!iKD_=Jey)Ba{jhE_#P_z=@7!3u-Km?@Ivy#C2Dq8+KMvEw
z%bz@gx^@G6^ZC~L`}Ho7&+me{+i+DKOAFWU^#Y{dw=pmYsj$0&kDTu-<qv=WZe+4Y
zVW)e^Z%el#XG2I(vhS>ep(F&RO!f!pY2u)1`8eIU;KUD$Ihun+Efu8wNT*k^RZ1Sr
zxD6Z3^9A(FVhOay)f(ov+9scaqfAM>vBV$2Xk&xi)m)mCq!HT3)(LbA4nRoUfk<y@
zTOydAZ~&+S>sXQ<c<NJ(+}zR?=-ZYSW_Gj(-pY6;h9a-(Y9_L3=4BOWKezK*=BOuG
ztvoYxS6D~fV_9tko3PI!r3|D5pi><lTnC8h!y94TPT~|U44kZwUi&Hx(s~%z$9}kj
zFrIjEqgL2DgrAd13M&P&)rp(^BuxWF+Z(i1#^WoaLk-d?O5(<S8&kzXFNp>OpM5_f
zc-%{5A)>F)21RxBRia>4;6rmt1-h@|7M)u-I*dbY_BT?Y2pF`QCVpFQ*mL{~qa)sf
z1jeWg@#G<zT;ZYF;9*fQ4>~W@M_4s6E8!vK7h~3s!4S?Egt3jGmZ|Omf{5JB9AJ*8
z-qjX(AWFtVtb|Ejm<ooa>~3n9!<*+AgEa2fQ>Y{g>Xa$^aKs%X3gi2?aq_7a8=m?B
zCJO#l39aPv>PJL=OvD)LL*icZ@1==8%5z$D&L$S7lzB>}jMRZ-qwZjC0=VScqzDTw
z{VT{=5gPDCunNBhB$OQd`lbd1kNg_BXXMu##>*{I4uDl)59=7p8iBSgZGg5S7)Du3
z#jaQGID+iTc>mnd6c?2eva!~aRRFeg2C!@NJ>tYU1K3mFC>FjU*aJ8X%$Lyl8rUN-
z&OHcYUkzIAq>sW?rDp(Cs1Klab!u2LngObNWRnrZe|I;~t=G0WAMH&HO@Nd>KNspX
zEOl1uvrsP#)GKnM|3cg=SoXzOT!8$$|5@DsDc&o}=Oe@~@pqE?jT*lk%jL30t&F>M
z`l3$c`$Q<KDHDs|BJSHnNO9^N5P%j%rTQTeV=5Dp=9}-(9G!+T$RPizDGlf^o}-3e
zVR=fiOo5491xk%-ffa=3+g$ww?<S=H1?D7EeTD(R0H7lIiOA`82Kd>hni%<sM&=GC
z(pA8ZMPWL=gXI9HePpbC6bkzuT9s~J4SQQHVoy`l9$pLRM)iKZbhG-SpsfKZDB!^0
z-dHQ4x&joXt66!SFc%9_V)NRbuVibKfy&agm1<m^QjJSeA#@KdE<^`?Cx*f%*N3Jz
z4vg0_oAJKiVDprYVSJjuLkMkrlJDT!xV6;~+glA(rWcwk=|L3uDqam9BgSIXeXFS1
zqwqm#O44I<nr?Kfqq@2|H~wN=Xj7mR4(h$$pu8vD!o+EQHYjK60|O=|ks4zeiC&R?
m;(vv=(M@DO3O1=kGFG35ou|<};klkWTdmF3mTJ?rxqkzZ60G+C

literal 0
HcmV?d00001

diff --git a/src/util/__pycache__/file.cpython-37.pyc b/src/util/__pycache__/file.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..79bb26389a52dc742a3327796d465b84ca436979
GIT binary patch
literal 1718
zcmbVM%Wfk@6s@XmyW6kCFoPKs!~{W*dSjyz?4k&U+2YOYS{hkSS2^yaAD*gmU@S)<
zCDP0$3%&qY@;fYR%Zgu^73Y?*!;D}<N%gq5Z{5ear}}ZP*Jc<$|M2^>IArV(61GF&
z@D-++W06eqf|WewrAP%QSrkDTreWDg8)Y+XmQfm&aT@z?LD5QEGL(%QmbPV6M%X(t
zmM!dE*_IvbJ=v8#?0a%g?qlDV!Bdv>|3oyF2p1K(u`<^#E~gg?2a_~hBR8aK=4O@`
z%7y23Ub&|F9;U`c_*6|xHO?q?8Ubbl^BYX_2#e%W+^{QQnGDu!z^*xGmY<7rNNc_p
zGQ16-VX{GABtt^YaYk-vkSm}BE!=&9MbZ^<07q8=&Tw%bxOh4AA;NVq69Yy_+<Mf!
z!&P)Nd^9`!_VF2fupe)RU$5@<Wc^)L)KjVC;gLBUVR0<sE>M-k)>!DgvOpj{E$Zx|
zT9!_n8W);;rP7YA?X;Mf)ucR4d)ZQJRoRXB8`JxU@%qcrye`#fT1l;@BXgM-%f+G|
zS!MFcY*|&R$g8uF(b;He^I}Bk4i_(6ywLTTR>s6|Vw#jL@A5uhJ=~@{@utT}YFe0+
z;nse&_vG(-;1{)^p)pxfe_UIuMXl|;t}k>8E|XAqu({?^7e#*RBCVb;m9ee^g;rMQ
z>QW_v3m~Tbsm|v4rSgHvoqh**eU+P~Pp)=wdb0>BUjpNC*yZnX{Q#OBO^7bkV^5RM
zaIom)HP>AtfaO+@<f-Cma=q;&A0u<}?cUpRbdt{|Rc$AplA2KSp%S6rC%f)8`Nh=E
z-_H5q4NTwgWT>cZ1uR4l^Z_)x3Ml*SG@n9v$zJlA@N|zYdj+CD!MqY{@eBWzakds<
zVld7)Y-tb#)ZTlT<|8a;Y|WkxuJ|iX*8zY4AmN(-B%X&)H*hCh(-pK<3_o{#hC@KF
zvt?f_R5n-H#a%;Qw~y&PXLY$aAtcNu;A&8q;f_3aCcujfz;Ey*2{dJ-ACg9>!h40q
zPN{v%MPUZydBf5NJm&fn$h#7$RC>L>%UvAMWpr2aTf#z&Q4U3bW4flv6e=Htp3q$_
zu0^Fx=$AxI@TD-W3HJXRjQ%YD5v=Wl(cAHT*+x5hI~w8TOS_K)m3GZd-{PmV`;&pL
z!qZg;KAa2xg@yV9eES&F5CC{D%xjXq6SeV8@wZ)6o1wo>W}5Q%WZ3xkyr(=o`8`!P
ppb|!!Quj{(@c=x+zK!v5S<7XiPN-j=_kI4Mh=aHlH{%BO;4i5RW;Or-

literal 0
HcmV?d00001

diff --git a/src/util/__pycache__/metrics.cpython-37.pyc b/src/util/__pycache__/metrics.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4bfc8ce38d1cf19588c27f7a8d413e46bc073359
GIT binary patch
literal 6430
zcmb_g+i%;}8Rt19CEAu9+i_#3X;LL=vMKBAWEU*hvcze+bXz)Oz={qq5^yvR<tULT
z=8#TfPr`sITigL1usv)q1Df<<!2Xv#?UAp0>RW-m^=ZHFBSp!I<;2Y>@Z5R6%kR6K
zqj#pKCk;HG{qEm){`H1o{D(UI$3$ZlZ~Qq5Whl`!*2Q~<lFEE+D6=QmrLvU$*jP7J
zNtIDs%25-jZ8fQ;P?uCiO`|TW8Fd7;6O`&g&8oS_Vts<!^Xe$tlWIY|hWROVOf90W
zs3mnA^|X3joj^UK-cTn|A5o{&o2X}1Rh>pXr_QLesOOdVl~H@^3-D>I`Be5F3yoF0
z@n2B{;9nWw`W(w3z<0*KmGf`s{Fie6%Q^o}&i@4SZh&9noA}N6uH<}A=X}rPd>_gA
zp3V85%lV$q`97-N0^jG;i5u-O`M}#~23mkm-$p?X)e`RwyfNT0I-+B^Mn|F)9TTPO
zSSU?pU>~tovr?Hvc*9iIQP;zoomz3wtZM>OlQuMMs!J&TX=n%4SC?1rwp+o97b+ci
zEAhidb1RD4D@hPH-1=4+2F*sexe{xCWh-ekS6V@$8-BbTZKsawHo`{Yy3^Pvrp*jd
z;@{4EA<}Yxkx#PGLv<n65;`5@9=&3bO39_U*cWL()@wqS@g4J@6%**D_GXZ{>L{B3
z#cS*w88E>!`*Tpi(g6pVB4tLE)N~Ln|1~Vow81DX|1h))T*lCvAnwC<;`)mPy`y_&
z<JrT|WP>1E7(<hFSwCLT8_{L`Ftk{{19Vx)S3gnE+ef~=G)Vw<j0H3l9OJ3+%vb=D
zPsKCi1X+(<+r$)}FR#_4CL7gLD8>kEMy`L7)-asY1NCBjkFW({TG%JNG)hqDx$(5h
zQ3mIm1--dpde4Jd(0oZ@kI=>k3tK;35Z@#07%jg*SaP~Cip_<spDpM;C+rxS^Z0@k
zeD5G!Ni_$FYDI1mohxY13@bW`_DA-m2(wimpkpE&eq#*b@D&Ji1V5N?J*w?bxc({<
zlMBB<)%EaOV+ao<rr2K?^ci&ePPAZjMMo%k-_oZjA9Muyv6Ap*bKuLlQ^VeJr>4Eo
zri_pZe>5qWNu<AxshIFGL`CcjnLFkRApVbR&43ao+CE#-eV(s+q%V!*3v)j4Ll4!Y
z`5SoSizpIKmL1Vgm}uom6D=f8sLn#mT7+UhwKx#gtbNm{t!eXpr*eJT?c{EtM|E0u
zT~8_3{XVh8e2SeRr_N__9^v!MNKsp$gG1=G0@qC^UANU%TTN<LT=&72*UV-#9ZcES
zvdGOU<*+tO6BJxExv8eeqbV$EPR7Tm#c7gG7W#E6=;)+RQ9);vp@L2gtOAN|@fj43
zm@GMR?9Xy6USAjuon;DJ{w6wulY|nKjre3!oN36a7oX2g`=cXL%MY5(5q?w0^ZhOD
z`P-L?=}i0jEZPNA#1M_M#e5Hqeh}@7ek)@HC0Ogqj%261i1Vdnmos5CR^R1}RYq`-
zOwG(<I1X`IH#O-lHN&k|YN{X{Wm(oQ{Rj(UzHEp|aZEUdBlUTV6-?7^1=EkwQR24j
z3NTlk4!dT@1f!<DKx4{;e$5jUGav?Dn10vlSiItW8silfR^XV7QK^5U;1~<|lF~Kr
zC3S)^S&Tsea2uZIL$$qDlSRNO8hiE8A6|vMb5dZ%5D0wEuGy@e^{F@t{6@XuH<InN
zqyyh;HdA?hZCzgf<*35Mt#|NwOeSmyM^=O*X7bWO2*WEiMff5}l+h`I#6xr_Nc03s
z(G&M&yp#a?p7@wXFy57kd}`8lg7k9ScqY1LF=FN;7LQ;D)gQ5O^)NdKEFF8%STee$
z#6rJ>ONgy34mI}L&gGkJ-SUzuPBB%#9mb6~3Bshh5hNc4L0C=nR#3%dqZ;WzHT)z{
z)uy)*G~?wj$eCER<HA*HHiIyoxgN)XP8#j-CtA04D#Iu><E<8ZX04n~5eYZjG@UKT
z^hYEvIt)%>zZq(S*ua;GlXpF>+!%Ks1gGrM<Hm`XKpb3uJ_M^V#Uw*40gie8%?kY;
zj1-^{AO$Fo&>0KLC>RWn8Uko(FGSWbMEN=%<8?M*ksTYbn962YO0NKmz5vd18~^g!
zf%s@n{%>PFr(C^EEpqBwNw1(v7o#?;K{wjkjdXj%+h{Z!Nh63eKuA~F{gmBk6p*-G
zPsReWfp239;{?b&A`HEnf?oj#ZCbz)p@SreD1*3gDuE{?@*WhfaQ^sI_HgFl7(=sH
z62Ws=OLOq3Y?hNOqS5Wd1_XK1t;=KZua7AvS%_Z}F^jJ4W<3k3tjroByZ5LO@2q|>
z#u=(`Al$NpxEkN}B2=CZs*P~SozBJ8bMbPmlv<%`w9=EV>)#Fh`);ws-*(|;$kSew
zFdFGp@F=1otdCQhH%=!%^qN}%`w^UuF`CLf2L2VN!i9U?7zt1M2T)!UZ%i53KtNyM
zw<0Vtix$uOq;7i1%}DL!b20fxV(gl`Q1NZ^6Y~?<wG#1EJ~5t%C-R9|w@As4odjW~
zC*FlBpV5Ci=30>_$ehSBDDU%jn?UN$g|65&cEv9K3Q@-Ifl(b%ZC4KWda@_}EFaK;
zNUX6$HJQrgi<&OE`0J#UIe9wwoU>~>b+K|XMNoYBYVt}Gp}=pqqO04)3h`Akc22iE
zU$<TFVW7RuVEIb3jW{1)MWeqOdM5p4m{_gA3sYNr;bwqDK>%dbu2E~1iXT%!663?c
z=zt}TR?~aPeuUNec^5J1UGxexXrsaj*3qGuL?(UP>>`yC8a<<HKl)*Uq)7<eS?-qT
zzouN3B~mQhBvUi@da_$av?(L~+7GI~rvInx9-)SRq|8iKQjVtMNX;25V`|gErp0h*
zzKM?nfaz9aNVUJkZn?wH%95B7XDBJ&#AxC0M3t<N2=mfRR!B(B2<{~7hR-2r50pPb
zxXjAEdFc<d4{d4){T|B05F*B2f)Gh~^U@!&xKHS(ha^O3^V#4aiApTdtv?OX`Pm`q
zus}6@bTAzf=+<9`==^+?P6-2y&@38#gc3Qpy9o9Jm=cS29;-O6_vwC@21XE@j$3Wq
zTyot0h82Bv-a>m1ZZA)sB|a4S49&j}k$?Z|lP`dMq4bp@>2Ez7qW+7oPo0G)xL=w&
zBma*f>T77%j;1zl3DI_TK;XD=mUd;m^8S<}m`D~nSZt<FE~m*IG})>qL(&ANCZ%eU
xO%sVBTVSi0z*y>B$*!4K>5Q4(^rmLsc1)-2EI1Ws(wVAQl_>}JJN)eI_dh+lhz<Y%

literal 0
HcmV?d00001

diff --git a/src/util/__pycache__/results.cpython-37.pyc b/src/util/__pycache__/results.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0ba8b91aa95213d71391809a1d6f15189a983d21
GIT binary patch
literal 2041
zcmah~OK&4Z5bmD09ouoTSsp@*4BB0-L9%&Bhyy}sg8~O+A$CtAp`kO~PR5xR-Q(=q
z+c|Bn`3D^ABRBpAKY_k*;KYR=uqUe8$r3GywAIx${iv$`>Z|s{ZnsUK{rcPQ-~Z(k
z@&|rgH5ZgGpsSBSaKdR$;>}27$9}u9XGcHwzare>?m6M^b0-cseME-dpKy*$Qj}Ni
zrOk&hBS=g+0bASL<=#1sUGDSXoW!2+X90E|PF)`H7OaK5&9`7J;vL?FvBi7fqi+V^
zR{0{&%A+~U_j9IHHqBDjWL5c{Q1iS|j}KDs(mmXeY(w9Nu6_ZbsU`{B3}fdWdgmef
zLtp5J_xSeFHteC=0qJNLq}zBie-&uDrjl#23VDAO;n$i{(zIr6P5BPry}%P&<=1w{
zt!q2*bVrjJS<^F5d-A7Cd)}HV&{tgznzht}U6k)#V3b>&z<qB|jFWK_6rwq<xCwK?
zWGSTaMR6!#zN(w7s7=UO!<1-DaF~@$E=((Dh|rNx#)BQgcsOCgf~B&Weh5moQ35-2
z0v>SU#Q0^^2(`TPO~p8XJP=Pa)u1xXaxs|7su(;wdqG=6YFs7i<@*OX|LknQE1?E(
zcynSj*p~vb0tZX+1kSNSjNnX|R{6Xr)o|N*Xi@G#R!ob{n`)ZR)v*arg*>d3Fir)b
z*Q_}<fsFw#BJ$h>(@GYsF;30n2z>A)Rj0<`Qve~8CHOH>!A=BxDAW29a5+HdL*IBx
z<WqSY9Lm>FypFD2*!MG$cR_#g>G-%R#F&*F?2grGme1?D8Ur3#GM$&DK-!I!Ovm#k
z%g1uVw^6+?QIce3)+9*?cGMdn2o306=mG7}KHYWtv`71-O_#%)_&d6?l{#n#8jjd4
zjdwvoGI9DQ(D}q!(KQmqnMka)p(Edh!MLnkfc`G%cHUlaBktk)F3S<SMUt{So#(6(
z9Dz~RC0%}eW03zoZ%En)6WDg3qizHK0h9)K4Yh~$<Y`|AI@FPF>9*d|9o^MEZL4nP
zaC+gK_2GK%%+r0WQd?C}g)?7Hpt6EBp>X|GsG$beP*D_KD13V+KrTNkxp?{*v`%(Z
zR#HGxU}D&kcOV86JQ6Y!%2IBFQo!n3K(>(%#<v8NZ-E}SnkrF^gme#gu7QZ}io{1n
z1}HGuGD6V;5qH@k00|FKg0&{MaG`^ui=u}DA0vs#NQB5x^=e*1#(5&ECrBY>pK3ay
zyEGsXT|T%uDVxJ7h11w@#BgmM&?FKEGC>e&i>~_;zXOTP0?xb#gS>^hYf*{=00U~w
zR?d)a;J|wMhfZ<?pUAr_{@lMYR15b1GhFr>AkFiu6w?U=XKU84Av4V-y^UL_qgP<G
zNqh(6vHt`B-}2s#5#j3sE*>8Y+gQ)|k|c>~CrMH9d5(G~Nq&TXoz058j}SZnVVtry
zZpljd0Zt(nHh^nES-7z9Bo-Kw$0j$T?Ox>W1(9p#hoQtInV^Ozh^Y;0{c>K^3;78y
b_$tjnM;rfT+(e&kV*MNwuaKD$wLj-?wEF~Y

literal 0
HcmV?d00001

diff --git a/src/util/evaluation.py b/src/util/evaluation.py
new file mode 100644
index 0000000..2ad6af6
--- /dev/null
+++ b/src/util/evaluation.py
@@ -0,0 +1,95 @@
+# from sklearn.externals.joblib import Parallel, delayed
+from joblib import Parallel, delayed
+from util.metrics import *
+from sklearn.metrics import f1_score
+import numpy as np
+import time
+
+def evaluation_metrics(y, y_):
+    if len(y.shape)==len(y_.shape)==1 and len(np.unique(y))>2: #single-label
+        raise NotImplementedError()#return f1_score(y,y_,average='macro'), f1_score(y,y_,average='micro')
+    else: #the metrics I implemented assume multiclass multilabel classification as binary classifiers
+        return macroF1(y, y_), microF1(y, y_), macroK(y, y_), microK(y, y_)
+
+def soft_evaluation_metrics(y, y_):
+    if len(y.shape)==len(y_.shape)==1 and len(np.unique(y))>2: #single-label
+        raise NotImplementedError()#return f1_score(y,y_,average='macro'), f1_score(y,y_,average='micro')
+    else: #the metrics I implemented assume multiclass multilabel classification as binary classifiers
+        return smoothmacroF1(y, y_), smoothmicroF1(y, y_), smoothmacroK(y, y_), smoothmicroK(y, y_)
+
+def evaluate(ly_true, ly_pred, metrics=evaluation_metrics, n_jobs=-1):
+    print('evaluation (n_jobs={})'.format(n_jobs))
+    if n_jobs == 1:
+        return {lang: metrics(ly_true[lang], ly_pred[lang]) for lang in ly_true.keys()}
+    else:
+        langs = list(ly_true.keys())
+        evals = Parallel(n_jobs=n_jobs)(delayed(metrics)(ly_true[lang], ly_pred[lang]) for lang in langs)
+        return {lang: evals[i] for i, lang in enumerate(langs)}
+
+def average_results(l_eval, show=True):
+    metrics  = []
+    for lang in l_eval.keys():
+        macrof1, microf1, macrok, microk = l_eval[lang]
+        metrics.append([macrof1, microf1, macrok, microk])
+        if show:
+            print('Lang %s: macro-F1=%.3f micro-F1=%.3f' % (lang, macrof1, microf1))
+
+    ave = np.mean(np.array(metrics), axis=0)
+    if show:
+        print('Averages: MF1, mF1, MK, mK', ave)
+    return ave
+
+
+def evaluate_method(polylingual_method, lX, ly, predictor=None, soft=False, return_time=False):
+    tinit=time.time()
+    print('prediction for test')
+    assert set(lX.keys()) == set(ly.keys()), 'inconsistent dictionaries in evaluate'
+    n_jobs = polylingual_method.n_jobs
+
+    if predictor is None:
+        predictor = polylingual_method.predict
+
+    metrics = evaluation_metrics
+    if soft is True:
+        metrics = soft_evaluation_metrics
+    ly_ = predictor(lX, ly)
+
+    eval_ = evaluate(ly, ly_, metrics=metrics, n_jobs=n_jobs)
+    if return_time:
+        return eval_, time.time()-tinit
+    else:
+        return eval_
+
+def evaluate_single_lang(polylingual_method, X, y, lang, predictor=None, soft=False):
+    print('prediction for test in a single language')
+    if predictor is None:
+        predictor = polylingual_method.predict
+
+    metrics = evaluation_metrics
+    if soft is True:
+        metrics = soft_evaluation_metrics
+
+    ly_ = predictor({lang:X})
+    return metrics(y, ly_[lang])
+
+def get_binary_counters(polylingual_method, lX, ly, predictor=None):
+    print('prediction for test')
+    assert set(lX.keys()) == set(ly.keys()), 'inconsistent dictionaries in evaluate'
+    n_jobs = polylingual_method.n_jobs
+    if predictor is None:
+        predictor = polylingual_method.predict
+    ly_ = predictor(lX)
+    print('evaluation (n_jobs={})'.format(n_jobs))
+    if n_jobs == 1:
+        return {lang: binary_counters(ly[lang], ly_[lang]) for lang in ly.keys()}
+    else:
+        langs = list(ly.keys())
+        evals = Parallel(n_jobs=n_jobs)(delayed(binary_counters)(ly[lang], ly_[lang]) for lang in langs)
+        return {lang: evals[i] for i, lang in enumerate(langs)}
+
+def binary_counters(y, y_):
+    y = np.reshape(y, (-1))
+    assert y.shape==y_.shape and len(y.shape)==1, 'error, binary vector expected'
+    counters = hard_single_metric_statistics(y, y_)
+    return counters.tp, counters.tn, counters.fp, counters.fn
+
diff --git a/src/util/file.py b/src/util/file.py
new file mode 100644
index 0000000..de6c4d0
--- /dev/null
+++ b/src/util/file.py
@@ -0,0 +1,36 @@
+from os import listdir, makedirs
+from os.path import isdir, isfile, join, exists, dirname
+#from sklearn.externals.six.moves import urllib
+import urllib
+
+
+def download_file(url, archive_filename):
+    def progress(blocknum, bs, size):
+        total_sz_mb = '%.2f MB' % (size / 1e6)
+        current_sz_mb = '%.2f MB' % ((blocknum * bs) / 1e6)
+        print('\rdownloaded %s / %s' % (current_sz_mb, total_sz_mb), end='')
+    print("Downloading %s" % url)
+    urllib.request.urlretrieve(url, filename=archive_filename, reporthook=progress)
+    print("")
+
+def download_file_if_not_exists(url, archive_path):
+    if exists(archive_path): return
+    makedirs_if_not_exist(dirname(archive_path))
+    download_file(url,archive_path)
+
+
+def ls(dir, typecheck):
+    el = [f for f in listdir(dir) if typecheck(join(dir, f))]
+    el.sort()
+    return el
+
+def list_dirs(dir):
+    return ls(dir, typecheck=isdir)
+
+def list_files(dir):
+    return ls(dir, typecheck=isfile)
+
+def makedirs_if_not_exist(path):
+    if not exists(path): makedirs(path)
+
+
diff --git a/src/util/metrics.py b/src/util/metrics.py
new file mode 100644
index 0000000..3efef4f
--- /dev/null
+++ b/src/util/metrics.py
@@ -0,0 +1,168 @@
+import numpy as np
+
+
+"""
+Scikit learn provides a full set of evaluation metrics, but they treat special cases differently.
+I.e., when the number of true positives, false positives, and false negatives ammount to 0, all
+affected metrices (precision, recall, and thus f1) output 0 in Scikit learn.
+We adhere to the common practice of outputting 1 in this case since the classifier has correctly
+classified all examples as negatives.
+"""
+
+class ContTable:
+    def __init__(self, tp=0, tn=0, fp=0, fn=0):
+        self.tp=tp
+        self.tn=tn
+        self.fp=fp
+        self.fn=fn
+
+    def get_d(self): return self.tp + self.tn + self.fp + self.fn
+
+    def get_c(self): return self.tp + self.fn
+
+    def get_not_c(self): return self.tn + self.fp
+
+    def get_f(self): return self.tp + self.fp
+
+    def get_not_f(self): return self.tn + self.fn
+
+    def p_c(self): return (1.0*self.get_c())/self.get_d()
+
+    def p_not_c(self): return 1.0-self.p_c()
+
+    def p_f(self): return (1.0*self.get_f())/self.get_d()
+
+    def p_not_f(self): return 1.0-self.p_f()
+
+    def p_tp(self): return (1.0*self.tp) / self.get_d()
+
+    def p_tn(self): return (1.0*self.tn) / self.get_d()
+
+    def p_fp(self): return (1.0*self.fp) / self.get_d()
+
+    def p_fn(self): return (1.0*self.fn) / self.get_d()
+
+    def tpr(self):
+        c = 1.0*self.get_c()
+        return self.tp / c if c > 0.0 else 0.0
+
+    def fpr(self):
+        _c = 1.0*self.get_not_c()
+        return self.fp / _c if _c > 0.0 else 0.0
+
+    def __add__(self, other):
+        return ContTable(tp=self.tp + other.tp, tn=self.tn + other.tn, fp=self.fp + other.fp, fn=self.fn + other.fn)
+
+def accuracy(cell):
+    return (cell.tp + cell.tn)*1.0 / (cell.tp + cell.fp + cell.fn + cell.tn)
+
+def f1(cell):
+    num = 2.0 * cell.tp
+    den = 2.0 * cell.tp + cell.fp + cell.fn
+    if den>0: return num / den
+    #we define f1 to be 1 if den==0 since the classifier has correctly classified all instances as negative
+    return 1.0
+
+def K(cell):
+    specificity, recall = 0., 0.
+
+    AN = cell.tn + cell.fp
+    if AN != 0:
+        specificity = cell.tn*1. / AN
+
+    AP = cell.tp + cell.fn
+    if AP != 0:
+        recall = cell.tp*1. / AP
+
+    if AP == 0:
+        return 2. * specificity - 1.
+    elif AN == 0:
+        return 2. * recall - 1.
+    else:
+        return specificity + recall - 1.
+
+#computes the (hard) counters tp, fp, fn, and tn fron a true and predicted vectors of hard decisions
+#true_labels and predicted_labels are two vectors of shape (number_documents,)
+def hard_single_metric_statistics(true_labels, predicted_labels):
+    assert len(true_labels)==len(predicted_labels), "Format not consistent between true and predicted labels."
+    nd = len(true_labels)
+    tp = np.sum(predicted_labels[true_labels==1])
+    fp = np.sum(predicted_labels[true_labels == 0])
+    fn = np.sum(true_labels[predicted_labels == 0])
+    tn = nd - (tp+fp+fn)
+    return ContTable(tp=tp, tn=tn, fp=fp, fn=fn)
+
+#computes the (soft) contingency table where tp, fp, fn, and tn are the cumulative masses for the posterioir
+# probabilitiesfron with respect to the true binary labels
+#true_labels and posterior_probabilities are two vectors of shape (number_documents,)
+def soft_single_metric_statistics(true_labels, posterior_probabilities):
+    assert len(true_labels)==len(posterior_probabilities), "Format not consistent between true and predicted labels."
+    tp = np.sum(posterior_probabilities[true_labels == 1])
+    fn = np.sum(1. - posterior_probabilities[true_labels == 1])
+    fp = np.sum(posterior_probabilities[true_labels == 0])
+    tn = np.sum(1. - posterior_probabilities[true_labels == 0])
+    return ContTable(tp=tp, tn=tn, fp=fp, fn=fn)
+
+#if the classifier is single class, then the prediction is a vector of shape=(nD,) which causes issues when compared
+#to the true labels (of shape=(nD,1)). This method increases the dimensions of the predictions.
+def __check_consistency_and_adapt(true_labels, predictions):
+    if predictions.ndim == 1:
+        return __check_consistency_and_adapt(true_labels, np.expand_dims(predictions, axis=1))
+    if true_labels.ndim == 1:
+        return __check_consistency_and_adapt(np.expand_dims(true_labels, axis=1),predictions)
+    if true_labels.shape != predictions.shape:
+        raise ValueError("True and predicted label matrices shapes are inconsistent %s %s."
+                         % (true_labels.shape, predictions.shape))
+    _,nC = true_labels.shape
+    return true_labels, predictions, nC
+
+def macro_average(true_labels, predicted_labels, metric, metric_statistics=hard_single_metric_statistics):
+    true_labels, predicted_labels, nC = __check_consistency_and_adapt(true_labels, predicted_labels)
+    return np.mean([metric(metric_statistics(true_labels[:, c], predicted_labels[:, c])) for c in range(nC)])
+
+def micro_average(true_labels, predicted_labels, metric, metric_statistics=hard_single_metric_statistics):
+    true_labels, predicted_labels, nC = __check_consistency_and_adapt(true_labels, predicted_labels)
+
+    accum = ContTable()
+    for c in range(nC):
+        other = metric_statistics(true_labels[:, c], predicted_labels[:, c])
+        accum = accum + other
+
+    return metric(accum)
+
+#true_labels and predicted_labels are two matrices in sklearn.preprocessing.MultiLabelBinarizer format
+def macroF1(true_labels, predicted_labels):
+    return macro_average(true_labels,predicted_labels, f1)
+
+#true_labels and predicted_labels are two matrices in sklearn.preprocessing.MultiLabelBinarizer format
+def microF1(true_labels, predicted_labels):
+    return micro_average(true_labels, predicted_labels, f1)
+
+#true_labels and predicted_labels are two matrices in sklearn.preprocessing.MultiLabelBinarizer format
+def macroK(true_labels, predicted_labels):
+    return macro_average(true_labels,predicted_labels, K)
+
+#true_labels and predicted_labels are two matrices in sklearn.preprocessing.MultiLabelBinarizer format
+def microK(true_labels, predicted_labels):
+    return micro_average(true_labels, predicted_labels, K)
+
+#true_labels is a matrix in sklearn.preprocessing.MultiLabelBinarizer format and posterior_probabilities is a matrix
+#of the same shape containing real values in [0,1]
+def smoothmacroF1(true_labels, posterior_probabilities):
+    return macro_average(true_labels,posterior_probabilities, f1, metric_statistics=soft_single_metric_statistics)
+
+#true_labels is a matrix in sklearn.preprocessing.MultiLabelBinarizer format and posterior_probabilities is a matrix
+#of the same shape containing real values in [0,1]
+def smoothmicroF1(true_labels, posterior_probabilities):
+    return micro_average(true_labels, posterior_probabilities, f1, metric_statistics=soft_single_metric_statistics)
+
+#true_labels is a matrix in sklearn.preprocessing.MultiLabelBinarizer format and posterior_probabilities is a matrix
+#of the same shape containing real values in [0,1]
+def smoothmacroK(true_labels, posterior_probabilities):
+    return macro_average(true_labels,posterior_probabilities, K, metric_statistics=soft_single_metric_statistics)
+
+#true_labels is a matrix in sklearn.preprocessing.MultiLabelBinarizer format and posterior_probabilities is a matrix
+#of the same shape containing real values in [0,1]
+def smoothmicroK(true_labels, posterior_probabilities):
+    return micro_average(true_labels, posterior_probabilities, K, metric_statistics=soft_single_metric_statistics)
+
diff --git a/src/util/results.py b/src/util/results.py
new file mode 100644
index 0000000..43529b4
--- /dev/null
+++ b/src/util/results.py
@@ -0,0 +1,33 @@
+import os
+import pandas as pd
+import numpy as np
+
+class PolylingualClassificationResults:
+    def __init__(self, file, autoflush=True, verbose=False):
+        self.file = file
+        self.columns = ['id', 'method', 'learner', 'embed', 'optimp', 'dataset', 'binary', 'languages', 'time', 'lang', 'macrof1', 'microf1', 'macrok', 'microk', 'notes']
+        self.autoflush = autoflush
+        self.verbose = verbose
+        if os.path.exists(file):
+            self.tell('Loading existing file from {}'.format(file))
+            self.df = pd.read_csv(file, sep='\t')
+        else:
+            self.tell('File {} does not exist. Creating new frame.'.format(file))
+            dir = os.path.dirname(self.file)
+            if dir and not os.path.exists(dir): os.makedirs(dir)
+            self.df = pd.DataFrame(columns=self.columns)
+
+    def already_calculated(self, id):
+        return (self.df['id'] == id).any()
+
+    def add_row(self, id, method, learner, embed, optimp, dataset, binary, ablation_lang, time, lang, macrof1, microf1, macrok=np.nan, microk=np.nan, notes=''):
+        s = pd.Series([id, method, learner, embed, optimp, dataset, binary, ablation_lang, time, lang, macrof1, microf1, macrok, microk, notes], index=self.columns)
+        self.df = self.df.append(s, ignore_index=True)
+        if self.autoflush: self.flush()
+        self.tell(s.to_string())
+
+    def flush(self):
+        self.df.to_csv(self.file, index=False, sep='\t')
+
+    def tell(self, msg):
+        if self.verbose: print(msg)