Merge pull request #17 from makcedward/dev

Dev

Author: makcedward

CHANGE.md

@@ -1,6 +1,12 @@
 NLPAUG Change Log
 ================
 
+**0.0.6** Jul 29, 2019:
+- Added new augmenter [TF-IDF based word replacement augmenter](https://arxiv.org/pdf/1904.12848.pdf)(TfIdfAug)
+- Added new augmenter [Spelling mistake simulation augmenter](https://arxiv.org/pdf/1711.02173.pdf)(SpellingAug)
+- Added new augmenter [Stopword Dropout augmenter](https://arxiv.org/pdf/1809.02079.pdf)(StopWordsAug)
+- Fixed [#14](https://github.com/makcedward/nlpaug/issues/14)
+
 **0.0.5** Jul 2, 2019:
 - Fixed [#3](https://github.com/makcedward/nlpaug/issues/3), [#4](https://github.com/makcedward/nlpaug/issues/4), [#5](https://github.com/makcedward/nlpaug/issues/5), [#7](https://github.com/makcedward/nlpaug/issues/7), [#10](https://github.com/makcedward/nlpaug/issues/10)
 

README.md

@@ -75,7 +75,7 @@ Download word2vec or GloVe files if you use `Word2VecAug`, `GloVeAug` or `Fastte
 
 ## Recent Changes
 
-**BETA** Jul 22, 2019:
+**0.0.6** Jul 29, 2019:
 - Added new augmenter [TF-IDF based word replacement augmenter](https://arxiv.org/pdf/1904.12848.pdf)(TfIdfAug)
 - Added new augmenter [Spelling mistake simulation augmenter](https://arxiv.org/pdf/1711.02173.pdf)(SpellingAug)
 - Added new augmenter [Stopword Dropout augmenter](https://arxiv.org/pdf/1809.02079.pdf)(StopWordsAug)

nlpaug/augmenter/char/char_augmenter.py

@@ -4,16 +4,19 @@
 
 
 class CharAugmenter(Augmenter):
-    def __init__(self, action, name='Char_Aug', aug_min=1, min_char=2, aug_char_p=0.3, aug_word_p=0.3, tokenizer=None, stopwords=[],
+    def __init__(self, action, name='Char_Aug', aug_min=1, min_char=2, aug_char_p=0.3, aug_word_p=0.3,
+                 tokenizer=None, reverse_tokenizer=None, stopwords=[],
                  verbose=0):
-        super(CharAugmenter, self).__init__(
+        super().__init__(
             name=name, method=Method.CHAR, action=action, aug_min=aug_min, verbose=verbose)
         self.aug_p = None
         self.aug_char_p = aug_char_p
         self.aug_word_p = aug_word_p
         self.min_char = min_char
         if tokenizer is not None:
             self.tokenizer = tokenizer
+        if reverse_tokenizer is not None:
+            self.reverse_tokenizer = reverse_tokenizer
         self.stopwords = stopwords
 
     def tokenizer(self, text):

nlpaug/augmenter/char/ocr.py

@@ -3,7 +3,8 @@
 
 
 class OcrAug(CharAugmenter):
-    def __init__(self, name='OCR_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3, stopwords=[], verbose=0):
+    def __init__(self, name='OCR_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, verbose=0):
         """
         Simulate OCR error on input text.
 
@@ -16,9 +17,9 @@ def __init__(self, name='OCR_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3, st
         :param stopwords: List of words which will be skipped from augment operation.
         :param verbose: Verbosity mode.
         """
-        super(OcrAug, self).__init__(
+        super().__init__(
             action=Action.SUBSTITUTE, name=name, aug_char_p=aug_char_p, aug_word_p=aug_word_p, aug_min=aug_min,
-            tokenizer=None, stopwords=stopwords, verbose=verbose)
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, stopwords=stopwords, verbose=verbose)
 
         self.model = self.get_model()
 

nlpaug/augmenter/char/qwerty.py

@@ -5,7 +5,8 @@
 
 
 class QwertyAug(CharAugmenter):
-    def __init__(self, name='Qwerty_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3, stopwords=[], verbose=0):
+    def __init__(self, name='Qwerty_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, verbose=0):
         """
         Simulate keyboard typo error on input text.
 
@@ -19,9 +20,9 @@ def __init__(self, name='Qwerty_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3,
         :param verbose: Verbosity mode.
         """
 
-        super(QwertyAug, self).__init__(
+        super().__init__(
             action=Action.SUBSTITUTE, name=name, aug_char_p=aug_char_p, aug_word_p=aug_word_p, aug_min=aug_min,
-            tokenizer=None, stopwords=stopwords, verbose=verbose)
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, stopwords=stopwords, verbose=verbose)
 
         self.model = self.get_model()
 

nlpaug/augmenter/char/random.py

@@ -7,7 +7,7 @@
 class RandomCharAug(CharAugmenter):
     def __init__(self, action=Action.SUBSTITUTE, name='RandomChar_Aug', aug_min=1, aug_char_p=0.3, aug_word_p=0.3,
                  include_upper_case=True, include_lower_case=True, include_numeric=True,
-                 spec_char='!@#$%^&*()_+', stopwords=[], verbose=0):
+                 spec_char='!@#$%^&*()_+', stopwords=[], tokenizer=None, reverse_tokenizer=None, verbose=0):
         """
         Apply random augment operation on input text
 
@@ -26,9 +26,9 @@ def __init__(self, action=Action.SUBSTITUTE, name='RandomChar_Aug', aug_min=1, a
         :param verbose: Verbosity mode.
         """
 
-        super(RandomCharAug, self).__init__(
+        super().__init__(
             action=action, name=name, aug_char_p=aug_char_p, aug_word_p=aug_word_p, aug_min=aug_min,
-            tokenizer=None, stopwords=stopwords, verbose=verbose)
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, stopwords=stopwords, verbose=verbose)
 
         self.include_upper_case = include_upper_case
         self.include_lower_case = include_lower_case

nlpaug/augmenter/word/bert.py

@@ -27,7 +27,7 @@ def init_bert_model(model_path, tokenizer_path, force_reload=False):
 class BertAug(WordAugmenter):
     def __init__(self, model_path='bert-base-uncased', tokenizer_path='bert-base-uncased', action=Action.SUBSTITUTE,
                  name='Bert_Aug', aug_min=1, aug_p=0.3, aug_n=5, stopwords=[], verbose=0):
-        super(BertAug, self).__init__(
+        super().__init__(
             action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=None, stopwords=stopwords,
             verbose=verbose)
         self.model_path = model_path

nlpaug/augmenter/word/fasttext.py

@@ -26,12 +26,13 @@ def init_fasttext_model(model_path, force_reload=False):
 
 class FasttextAug(WordEmbsAugmenter):
     def __init__(self, model_path='.', model=None, action=Action.SUBSTITUTE,
-                 name='Fasttext_Aug', aug_min=1, aug_p=0.3, aug_n=5, tokenizer=None, stopwords=[], force_reload=False,
+                 name='Fasttext_Aug', aug_min=1, aug_p=0.3, aug_n=5, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, force_reload=False,
                  verbose=0):
         super().__init__(
             model_path=model_path, aug_n=aug_n,
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         if model is None:
             self.model = self.get_model(force_reload=force_reload)

nlpaug/augmenter/word/glove.py

@@ -26,12 +26,13 @@ def init_glove_model(model_path, force_reload=False):
 
 class GloVeAug(WordEmbsAugmenter):
     def __init__(self, model_path='.', model=None, action=Action.SUBSTITUTE,
-                 name='GloVe_Aug', aug_min=1, aug_p=0.3, aug_n=5, tokenizer=None, stopwords=[], force_reload=False,
+                 name='GloVe_Aug', aug_min=1, aug_p=0.3, aug_n=5, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, force_reload=False,
                  verbose=0):
-        super(GloVeAug, self).__init__(
+        super().__init__(
             model_path=model_path, aug_n=aug_n,
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         if model is None:
             self.model = self.get_model(force_reload=force_reload)

nlpaug/augmenter/word/random.py

@@ -3,11 +3,11 @@
 
 
 class RandomWordAug(WordAugmenter):
-    def __init__(self, action=Action.DELETE, name='RandomWord_Aug', aug_min=1, aug_p=0.3, tokenizer=None, stopwords=[],
-                 verbose=0):
-        super(RandomWordAug, self).__init__(
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+    def __init__(self, action=Action.DELETE, name='RandomWord_Aug', aug_min=1, aug_p=0.3, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, verbose=0):
+        super().__init__(
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
     def swap(self, text):
         """

nlpaug/augmenter/word/spelling.py

@@ -20,11 +20,11 @@ def init_spelling_error_model(dict_path, include_reverse, force_reload=False):
 
 
 class SpellingAug(WordAugmenter):
-    def __init__(self, dict_path, name='Spelling_Aug', aug_min=1, aug_p=0.3, tokenizer=None, stopwords=[],
-                 include_reverse=True, verbose=0):
+    def __init__(self, dict_path, name='Spelling_Aug', aug_min=1, aug_p=0.3, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, include_reverse=True, verbose=0):
         super().__init__(
-            action=Action.SUBSTITUTE, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=Action.SUBSTITUTE, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         self.dict_path = dict_path
         self.include_reverse = include_reverse

nlpaug/augmenter/word/stopwords.py

@@ -3,15 +3,15 @@
 
 
 class StopWordsAug(WordAugmenter):
-    def __init__(self, stopwords, action=Action.DELETE, name='StopWords_Aug', aug_min=1, aug_p=0.3, tokenizer=None,
-                 case_sensitive=False, verbose=0):
+    def __init__(self, stopwords, action=Action.DELETE, name='StopWords_Aug', aug_min=1, aug_p=0.3,
+                 tokenizer=None, reverse_tokenizer=None, case_sensitive=False, verbose=0):
 
         if not case_sensitive:
             stopwords = [t.lower() for t in stopwords]
 
         super().__init__(
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         self.case_sensitive = case_sensitive
 

nlpaug/augmenter/word/tfidf.py

@@ -23,11 +23,11 @@ def init_tfidf_model(model_path, force_reload=False):
 
 class TfIdfAug(WordAugmenter):
     def __init__(self, model_path='.', action=Action.SUBSTITUTE,
-                 name='TfIdf_Aug', aug_min=1, aug_p=0.3, aug_n=5, tokenizer=None, n_gram_separator='_',
-                 stopwords=[], verbose=0):
+                 name='TfIdf_Aug', aug_min=1, aug_p=0.3, aug_n=5, n_gram_separator='_',
+                 stopwords=[], tokenizer=None, reverse_tokenizer=None, verbose=0):
         super().__init__(
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
         self.model_path = model_path
         self.aug_n = aug_n
         self.model = self.get_model(force_reload=False)

nlpaug/augmenter/word/word2vec.py

@@ -25,12 +25,12 @@ def init_word2vec_model(model_path, force_reload=False):
 
 class Word2vecAug(WordEmbsAugmenter):
     def __init__(self, model_path='.', model=None, action=Action.SUBSTITUTE,
-                 name='Word2vec_Aug', aug_min=1, aug_p=0.3, aug_n=5, tokenizer=None, stopwords=[], force_reload=False,
-                 verbose=0):
-        super(Word2vecAug, self).__init__(
+                 name='Word2vec_Aug', aug_min=1, aug_p=0.3, aug_n=5, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, force_reload=False, verbose=0):
+        super().__init__(
             model_path=model_path, aug_n=aug_n,
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         if model is None:
             self.model = self.get_model(force_reload=force_reload)

nlpaug/augmenter/word/word_augmenter.py

@@ -4,12 +4,15 @@
 
 
 class WordAugmenter(Augmenter):
-    def __init__(self, action, name='Word_Aug', aug_min=1, aug_p=0.3, tokenizer=None, stopwords=[], verbose=0):
-        super(WordAugmenter, self).__init__(
+    def __init__(self, action, name='Word_Aug', aug_min=1, aug_p=0.3, stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, verbose=0):
+        super().__init__(
             name=name, method=Method.WORD, action=action, aug_min=aug_min, verbose=verbose)
         self.aug_p = aug_p
         if tokenizer is not None:
             self.tokenizer = tokenizer
+        if reverse_tokenizer is not None:
+            self.reverse_tokenizer = reverse_tokenizer
         self.stopwords = stopwords
 
     def tokenizer(self, text):

nlpaug/augmenter/word/word_embs_aug.py

@@ -8,11 +8,11 @@
 
 class WordEmbsAugmenter(WordAugmenter):
     def __init__(self, model_path='.', action=Action.SUBSTITUTE,
-                 name='WordEmbs_Aug', aug_min=1, aug_p=0.3, aug_n=5, tokenizer=None, n_gram_separator='_',
-                 stopwords=[], verbose=0):
-        super(WordEmbsAugmenter, self).__init__(
-            action=action, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+                 name='WordEmbs_Aug', aug_min=1, aug_p=0.3, aug_n=5, n_gram_separator='_',
+                 stopwords=[], tokenizer=None, reverse_tokenizer=None, verbose=0):
+        super().__init__(
+            action=action, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
         self.model_path = model_path
         self.aug_n = aug_n
         self.model = self.get_model(force_reload=False)

nlpaug/augmenter/word/wordnet.py

@@ -6,10 +6,11 @@
 
 
 class WordNetAug(WordAugmenter):
-    def __init__(self, name='WordNet_Aug', aug_min=1, aug_p=0.3, lang='eng', tokenizer=None, stopwords=[], verbose=0):
+    def __init__(self, name='WordNet_Aug', aug_min=1, aug_p=0.3, lang='eng', stopwords=[],
+                 tokenizer=None, reverse_tokenizer=None, verbose=0):
         super().__init__(
-            action=Action.SUBSTITUTE, name=name, aug_p=aug_p, aug_min=aug_min, tokenizer=tokenizer, stopwords=stopwords,
-            verbose=verbose)
+            action=Action.SUBSTITUTE, name=name, aug_p=aug_p, aug_min=aug_min, stopwords=stopwords,
+            tokenizer=tokenizer, reverse_tokenizer=reverse_tokenizer, verbose=verbose)
 
         self.model = self.get_model()
         self.lang = lang

nlpaug/util/__init__.py

@@ -3,7 +3,7 @@
 from nlpaug.util.method import *
 from nlpaug.util.exception import *
 from nlpaug.util.math import *
-
+from nlpaug.util.text import *
 
 from nlpaug.util.part_of_speech import *
 

nlpaug/util/text/__init__.py

@@ -0,0 +1,16 @@
+import re
+
+ADDING_SPACE_AROUND_PUNCTUATION_REGEX = re.compile(r'(?<! )(?=[.,!?()])|(?<=[.,!?()])(?! )')
+SPLIT_WORD_REGEX = re.compile(r'\b.*?\S.*?(?:\b|$)')
+
+# re.compile(r"(\W+)")
+# re.compile(r"\w+|[^\w\s]")
+
+
+def add_space_around_punctuation(text):
+    return ADDING_SPACE_AROUND_PUNCTUATION_REGEX.sub(r' ', text)
+
+
+def split_sentence(text):
+    return SPLIT_WORD_REGEX.findall(text)
+

nlpaug/util/text/tokenizer.py

@@ -0,0 +1,38 @@
+import unittest
+
+import nlpaug.augmenter.char as nac
+import nlpaug.util.text.tokenizer as text_tokenizer
+
+
+class TestCharacter(unittest.TestCase):
+    def test_empty(self):
+        texts = ['', None]
+
+        augs = [
+            nac.OcrAug(),
+            nac.QwertyAug(),
+        ]
+
+        for text in texts:
+            for aug in augs:
+                augmented_text = aug.augment(text)
+                self.assertEqual(text, augmented_text)
+
+    def test_tokenizer(self):
+        augs = [
+            nac.OcrAug(tokenizer=text_tokenizer.split_sentence),
+            nac.QwertyAug(tokenizer=text_tokenizer.split_sentence),
+            nac.RandomCharAug(tokenizer=text_tokenizer.split_sentence),
+        ]
+
+        text = 'The quick brown fox, jumps over lazy dog.'
+        expected_tokens = ['The', ' quick', ' brown', ' fox', ', ', 'jumps', ' over', ' lazy', ' dog', '.']
+        for aug in augs:
+            tokens = aug.tokenizer(text)
+            self.assertEqual(tokens, expected_tokens)
+
+        text = 'The quick !brown fox, jumps # over lazy dog .'
+        expected_tokens = ['The', ' quick', ' !', 'brown', ' fox', ', ', 'jumps', ' # ', 'over', ' lazy', ' dog', ' .']
+        for aug in augs:
+            tokens = aug.tokenizer(text)
+            self.assertEqual(tokens, expected_tokens)

test/augmenter/char/test_char.py

@@ -39,10 +39,3 @@ def test_ocr_multi_words(self):
             self.assertTrue(is_augmented)
 
         self.assertTrue(len(texts) > 0)
-
-    def test_ocr_empty(self):
-        texts = ['', None]
-        aug = OcrAug()
-        for text in texts:
-            augmented_text = aug.augment(text)
-            self.assertEqual(text, augmented_text)

test/augmenter/char/test_ocr.py

@@ -21,10 +21,3 @@ def test_qwerty_multi_words(self):
             self.assertNotEqual(text, augmented_text)
 
         self.assertTrue(len(texts) > 0)
-
-    def test_qwerty_empty(self):
-        texts = ['', None]
-        aug = QwertyAug()
-        for text in texts:
-            augmented_text = aug.augment(text)
-            self.assertEqual(text, augmented_text)

test/augmenter/char/test_qwerty.py

@@ -14,14 +14,6 @@ def setUpClass(cls):
             os.path.dirname(__file__), '..', '..', '..', '.env'))
         load_dotenv(env_config_path)
 
-    def test_empty_input_for_insert(self):
-        text = ' '
-
-        aug = naw.BertAug(action=Action.INSERT)
-        augmented_text = aug.augment(text)
-
-        self.assertEqual(augmented_text, '')
-
     def test_oov(self):
         unknown_token = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
         texts = [