merging

2020-01-09 10:50:35 +01:00 · 2020-01-09 10:50:35 +01:00 · e96968fc45
parent a8d76b6f52
commit e96968fc45
16 changed files with 11 additions and 0 deletions
--- a/src/data/pycache/init.cpython-37.pyc
+++ b/src/data/pycache/init.cpython-37.pyc
--- a/src/data/pycache/embeddings.cpython-37.pyc
+++ b/src/data/pycache/embeddings.cpython-37.pyc
--- a/src/data/pycache/languages.cpython-37.pyc
+++ b/src/data/pycache/languages.cpython-37.pyc
--- a/src/data/pycache/supervised.cpython-37.pyc
+++ b/src/data/pycache/supervised.cpython-37.pyc
--- a/src/data/pycache/text_preprocessor.cpython-37.pyc
+++ b/src/data/pycache/text_preprocessor.cpython-37.pyc
--- a/src/data/pycache/tsr_function__.cpython-37.pyc
+++ b/src/data/pycache/tsr_function__.cpython-37.pyc
--- a/src/data/reader/pycache/init.cpython-37.pyc
+++ b/src/data/reader/pycache/init.cpython-37.pyc
--- a/src/data/reader/pycache/jrcacquis_reader.cpython-37.pyc
+++ b/src/data/reader/pycache/jrcacquis_reader.cpython-37.pyc
--- a/src/data/reader/pycache/rcv_reader.cpython-37.pyc
+++ b/src/data/reader/pycache/rcv_reader.cpython-37.pyc
--- a/src/data/reader/pycache/wikipedia_tools.cpython-37.pyc
+++ b/src/data/reader/pycache/wikipedia_tools.cpython-37.pyc
--- a/src/learning/pycache/learners.cpython-37.pyc
+++ b/src/learning/pycache/learners.cpython-37.pyc
--- a/src/util/pycache/evaluation.cpython-37.pyc
+++ b/src/util/pycache/evaluation.cpython-37.pyc
--- a/src/util/pycache/file.cpython-37.pyc
+++ b/src/util/pycache/file.cpython-37.pyc
--- a/src/util/pycache/metrics.cpython-37.pyc
+++ b/src/util/pycache/metrics.cpython-37.pyc
--- a/src/util/pycache/results.cpython-37.pyc
+++ b/src/util/pycache/results.cpython-37.pyc
--- a/src/util/util.py
+++ b/src/util/util.py
@ -0,0 +1,11 @@
+from tqdm import tqdm
+import re
+import sys
+
+
+def mask_numbers(data, number_mask='numbermask'):
+    mask = re.compile(r'\b[0-9][0-9.,-]*\b')
+    masked = []
+    for text in tqdm(data, desc='masking numbers'):
+        masked.append(mask.sub(number_mask, text))
+    return masked