Update preprocess: now returns generator objects

test/test_text.py

@@ -13,7 +13,7 @@ class TestText(TestCase):
         )
         expected = ""
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_1(self):
         text = "<div>Hello, <strong>world</strong></div>"
@@ -23,7 +23,7 @@ class TestText(TestCase):
         )
         expected = "Hello, world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_2(self):
         text = "<div>Hello, <strong>world</strong></div>"
@@ -34,18 +34,7 @@ class TestText(TestCase):
         )
         expected = "hello, world"
 
-        self.assertEqual(cleaned, expected)
-
-    def test_html_3(self):
-        text = "<div>\n Hello, \t\n<strong> world    </strong>\n\t</div>"
-        cleaned = preprocess(
-            text,
-            clean_html=True,
-            lowercase=True,
-        )
-        expected = " hello, world "
-
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_4(self):
         text = "<div>\n Hello, \t\n<strong> world    </strong>\n\t</div>"
@@ -53,11 +42,10 @@ class TestText(TestCase):
             text,
             clean_html=True,
             lowercase=True,
-            strip=True
         )
         expected = "hello, world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_5(self):
         text = "<div>\n Hello, \t\n<strong> world    </strong>\n\t</div>"
@@ -65,12 +53,11 @@ class TestText(TestCase):
             text,
             clean_html=True,
             lowercase=True,
-            strip=True,
             remove_punctuation=True
         )
         expected = "hello world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_6(self):
         text = "<div>\n Hello, \t\n<strong>a the world    </strong>\n\t</div>"
@@ -79,12 +66,11 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             remove_stopwords_en=True
         )
         expected = "hello world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_7(self):
         text = "<div>\n Hello, \t\n<strong>a the worlds    </strong>\n\t</div>"
@@ -93,13 +79,12 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             remove_stopwords_en=True,
             lemmatize=True
         )
         expected = "hello world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_8(self):
         text = "<div>\n Hello, \t\n<strong>a the worlds!    </strong>\n\t</div>"
@@ -108,13 +93,12 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             remove_stopwords_en=True,
             lemmatize=True
         )
         expected = "hello world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_9(self):
         text = "<div>\n Hello, \t\n<strong>world! it's it`s   </strong>\n\t</div>"
@@ -123,13 +107,12 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=True,
             fix_single_quotes=True
         )
         expected = "hello world it's it's"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
     
     def test_single_quote(self):
         text = "it's it`s it’s"
@@ -140,7 +123,7 @@ class TestText(TestCase):
         )
         expected = "it's it's it's"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_10(self):
         text = "<div>\n Hello, \t\n<strong>world! it's it`s https://google.ca/test/abc.pdf  </strong>\n\t</div>"
@@ -149,14 +132,13 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=True,
             fix_single_quotes=True,
             remove_urls=True
         )
         expected = "hello world it's it's"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_11(self):
         text = "<div>\n Hello, \t\n<strong>world! it's it`s & | </strong>\n\t</div>"
@@ -165,7 +147,6 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=True,
             fix_single_quotes=True,
             remove_stopwords_en=True,
@@ -173,7 +154,7 @@ class TestText(TestCase):
         )
         expected = "hello world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_no_root(self):
         text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a<wbr>servant that is against civilization and humanity?<br>Literally instant summon."
@@ -183,7 +164,6 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=False,
             fix_single_quotes=True,
             remove_stopwords_en=False,
@@ -191,7 +171,7 @@ class TestText(TestCase):
         )
 
         expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_entity(self):
         text = "doesn&#039;t"
@@ -201,7 +181,6 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=False,
             fix_single_quotes=True,
             remove_stopwords_en=False,
@@ -209,7 +188,7 @@ class TestText(TestCase):
         )
 
         expected = "doesn't"
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_html_invalid_attribute(self):
         text = '<root><iframe width="560" height="315" src=" " title="youtube video player" frameborder="0" allowfullscreen></iframe></root>'
@@ -219,7 +198,6 @@ class TestText(TestCase):
             clean_html=True,
             lowercase=True,
             remove_punctuation=True,
-            strip=True,
             lemmatize=False,
             fix_single_quotes=True,
             remove_stopwords_en=False,
@@ -228,7 +206,7 @@ class TestText(TestCase):
 
         expected = ""
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_bigrams(self):
         text = "x A b c d e f g h"
@@ -243,7 +221,7 @@ class TestText(TestCase):
         )
         expected = "x a_b c_d e f_g h"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_trigrams(self):
         text = "x A b c d e f g h"
@@ -257,7 +235,7 @@ class TestText(TestCase):
         )
         expected = "x a_b_c d e_f_g h"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)
 
     def test_remove_numbers(self):
         text = "Hello1 test1124test 12 1 1111111 world"
@@ -268,4 +246,4 @@ class TestText(TestCase):
         )
         expected = "hello1 test1124test world"
 
-        self.assertEqual(cleaned, expected)
+        self.assertEqual(" ".join(cleaned), expected)